{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分类训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x00\\x00\\x00\\x02'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import struct               # 打包解包模块  \n",
    "struct.pack('>i',2)         # 将2 打包成int类型占4个字节，pack打包成一个二进制流、或者byte数组或叫二进制的串"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![jupyter](./g1.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2051, 60000, 28, 28)\n",
      "47040000\n"
     ]
    }
   ],
   "source": [
    "import struct \n",
    "with open('./MNIST_data/train-images-idx3-ubyte','rb') as f :\n",
    "    buffer = f.read(4*4)                       # 4个int，读16个字节\n",
    "    head = struct.unpack('>iiii',buffer)      # unpack解包出4个int类型\n",
    "    print(head)\n",
    "    length = head[1] * head[2] * head[3]      # 读取60000张图片，每张图片有28行，28列\n",
    "    print(length)\n",
    "    buffer = f.read(length)                   # 读取像素包\n",
    "    data = struct.unpack('>{}B'.format(length),buffer)        # 读取出length个byte字节"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47040000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tuple"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "imgs = np.reshape(data,(head[1],head[2],head[3]))    # 将data 重新塑性成三维的矩阵\n",
    "imgs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "for i in range(5):\n",
    "    plt.imshow(imgs[i],cmap = 'gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\utils\\deprecation.py:85: DeprecationWarning: Function fetch_mldata is deprecated; fetch_mldata was deprecated in version 0.20 and will be removed in version 0.22. Please use fetch_openml.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\utils\\deprecation.py:85: DeprecationWarning: Function mldata_filename is deprecated; mldata_filename was deprecated in version 0.20 and will be removed in version 0.22. Please use fetch_openml.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# 机器学习方法中的解包，可从网络上或者本地直接抓取数据，fetch_mldata的第一个参数可为url地址，第二个参数为本地文件位置\n",
    "\n",
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original',data_home = './')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'COL_NAMES': ['label', 'data'],\n",
       " 'target': array([0., 0., 0., ..., 9., 9., 9.]),\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8)}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist         # 为字典，4个键值对"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   * DESCR 数据集描述\n",
    "   * data 包含一个（二维）数组，每个实例为一行，每个特征为一列 X\n",
    "   * target 包含一个带有标签的数组 y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将标签和训练数据拆分出来\n",
    "\n",
    "X , y = mnist['data'],mnist['target'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_digit = X[36000]                            # 抽取36000该行数据有784个数据\n",
    "some_digit_image = some_digit.reshape(28,28)    # 转换成二维数组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANpElEQVR4nO3db6xU9Z3H8c9XLQ+UJoB3NCAutzZg1piUkgnZxE1jbbZRicE+qMIDZJMmtw/EQMSkpE2shiekrjY1MU3oQnpduzaYlgUj2a3BJoQHVkcDgiVFivyrN9wBEnv7gHSx3z64x+YCc37nMufMnIHv+5VMZuZ858z5Zrgfzsz5zZmfubsAXPuuq7sBAP1B2IEgCDsQBGEHgiDsQBA39HNjQ0NDPjw83M9NAqEcO3ZMZ86csU61UmE3s/sl/UTS9ZL+0903pR4/PDysVqtVZpMAEprNZm6t67fxZna9pJckPSDpLkkrzeyubp8PQG+V+cy+VNIRdz/q7n+V9EtJy6tpC0DVyoT9Nkknp9w/lS27iJmNmFnLzFrtdrvE5gCUUSbsnQ4CXPbdW3ff7O5Nd282Go0SmwNQRpmwn5J0+5T78yV9Uq4dAL1SJuzvSlpoZl8ysxmSVkjaWU1bAKrW9dCbu18wszWS/k+TQ29b3f3DyjoDUKlS4+zuvkvSrop6AdBDfF0WCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIErN4grUadu2bcn6gQMHcmsvv/xy1e1c5Pjx4z19/m6UCruZHZM0IekzSRfcvVlFUwCqV8We/evufqaC5wHQQ3xmB4IoG3aX9Bsze8/MRjo9wMxGzKxlZq12u11ycwC6VTbs97j7EkkPSHrczL526QPcfbO7N9292Wg0Sm4OQLdKhd3dP8muxyVtl7S0iqYAVK/rsJvZTWb2xc9vS/qmpINVNQagWmWOxt8qabuZff48/+3u/1tJV7hmTExM5Nb27t2bXHfjxo3J+ttvv52sZ3+byHQddnc/KukrFfYCoIcYegOCIOxAEIQdCIKwA0EQdiAITnG9xl24cCFZHxsbK/X8RcNjH3/8cW7trbfeKrXtXhoaGkrWV6xY0adOqsOeHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCYJz9Glc0jj48PJysu3uyPsinkS5evDi3tmrVquS6y5YtS9YXLlzYVU91Ys8OBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzn6Ne+qpp5L1onH0onqRefPm5dZGRjrOGPYPTz/9dKlt42Ls2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZrwFbt27Nre3atSu5btnz0YvWP3v2bG6t6DftDx8+nKwvWrQoWcfFCvfsZrbVzMbN7OCUZXPM7E0z+yi7nt3bNgGUNZ238T+XdP8lyzZI2u3uCyXtzu4DGGCFYXf3PZLOXbJ4uaTR7PaopIcr7gtAxbo9QHeru49JUnZ9S94DzWzEzFpm1mq3211uDkBZPT8a7+6b3b3p7s1Go9HrzQHI0W3YT5vZXEnKrserawlAL3Qb9p2SVme3V0vaUU07AHrFpvG74K9KulfSkKTTkn4o6X8kbZP0T5JOSPq2u196EO8yzWbTW61WyZbjSY2jS9KTTz6ZW5uYmCi17Tp/N37BggXJ+tGjR3u27atVs9lUq9Xq+I9S+KUad1+ZU/pGqa4A9BVflwWCIOxAEIQdCIKwA0EQdiAITnG9Cjz77LPJepnhtVmzZiXrM2fOTNavuy69vzh//nxubXw8/V2s48ePJ+u4MuzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtmvAsuXL0/WX3rppdza6tWrc2uStGbNmmR9yZIlyXqRsbGx3NqyZcuS6+7fv7/UtnEx9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7FeBF198sVS9Tqmfoi76meqiOq4Me3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIJx9szJkyeT9RtvvDG3dvPNN1fdzjUjdU560XTPRfUdO3Yk60W/AxBN4Z7dzLaa2biZHZyy7Bkz+5OZ7csuD/a2TQBlTedt/M8l3d9h+Y/dfXF22VVtWwCqVhh2d98j6VwfegHQQ2UO0K0xsw+yt/mz8x5kZiNm1jKzVrvdLrE5AGV0G/afSvqypMWSxiQ9n/dAd9/s7k13bzYajS43B6CsrsLu7qfd/TN3/5ukn0laWm1bAKrWVdjNbO6Uu9+SdDDvsQAGQ+E4u5m9KuleSUNmdkrSDyXda2aLJbmkY5K+28MeK7Fp06ZkfXR0NFmfMWNGbu2OO+5Irrt9+/Zk/Wp29uzZZH3Dhg25tYMH0/uI4eHhblpCjsKwu/vKDou39KAXAD3E12WBIAg7EARhB4Ig7EAQhB0IIswpru+8806yfvjw4a6f+8SJE8n6+vXrk/Xnn8/9AmLtik79feONN5L11PDaDTek//zuvvvuZJ1TWK8Me3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCCLMOHsvzZo1K1kf5HH0ImvXrk3Wi37OOWXevHk9e25cjj07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgQRZpy96GeJZ86cmaxPTEzk1h566KFuWuqLRx99NFl/7bXXknV3T9aLplVOee6557peF1eOPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBBFmnP2FF15I1o8cOZKsp34f/fz588l1i8ayi2zcuDFZ//TTT3Nr586dS65bNE5+5513JuuPPfZY1/U5c+Yk10W1CvfsZna7mf3WzA6Z2YdmtjZbPsfM3jSzj7Lr2b1vF0C3pvM2/oKk9e7+z5L+RdLjZnaXpA2Sdrv7Qkm7s/sABlRh2N19zN3fz25PSDok6TZJyyWNZg8blfRwr5oEUN4VHaAzs2FJX5X0O0m3uvuYNPkfgqRbctYZMbOWmbXa7Xa5bgF0bdphN7OZkn4laZ27/3m667n7Zndvunuz0Wh00yOACkwr7Gb2BU0G/Rfu/uts8Wkzm5vV50oa702LAKpQOPRmk2MzWyQdcvep41c7Ja2WtCm7vqp/93fdunXJempa5t27dyfX3bJlS7Ley9NIFy1alKwPDQ0l66+88kqyvmDBgivuCfWYzjj7PZJWSTpgZvuyZd/XZMi3mdl3JJ2Q9O3etAigCoVhd/e9kvJ2Ld+oth0AvcLXZYEgCDsQBGEHgiDsQBCEHQgizCmuRe67775kPTWWXnQa6f79+5P1PXv2JOuvv/56sv7EE0/k1h555JHkuvPnz0/Wce1gzw4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQVjRudRVajab3mq1+rY9IJpms6lWq9XxLFX27EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxBEYdjN7HYz+62ZHTKzD81sbbb8GTP7k5ntyy4P9r5dAN2aziQRFyStd/f3zeyLkt4zszez2o/d/T961x6AqkxnfvYxSWPZ7QkzOyTptl43BqBaV/SZ3cyGJX1V0u+yRWvM7AMz22pms3PWGTGzlpm12u12qWYBdG/aYTezmZJ+JWmdu/9Z0k8lfVnSYk3u+Z/vtJ67b3b3prs3G41GBS0D6Ma0wm5mX9Bk0H/h7r+WJHc/7e6fufvfJP1M0tLetQmgrOkcjTdJWyQdcvcXpiyfO+Vh35J0sPr2AFRlOkfj75G0StIBM9uXLfu+pJVmtliSSzom6bs96RBAJaZzNH6vpE6/Q72r+nYA9ArfoAOCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRh7t6/jZm1JR2fsmhI0pm+NXBlBrW3Qe1LorduVdnbAnfv+PtvfQ37ZRs3a7l7s7YGEga1t0HtS6K3bvWrN97GA0EQdiCIusO+uebtpwxqb4Pal0Rv3epLb7V+ZgfQP3Xv2QH0CWEHgqgl7GZ2v5n9wcyOmNmGOnrIY2bHzOxANg11q+ZetprZuJkdnLJsjpm9aWYfZdcd59irqbeBmMY7Mc14ra9d3dOf9/0zu5ldL+mwpH+TdErSu5JWuvvv+9pIDjM7Jqnp7rV/AcPMvibpL5Jedve7s2U/knTO3Tdl/1HOdvfvDUhvz0j6S93TeGezFc2dOs24pIcl/btqfO0SfT2iPrxudezZl0o64u5H3f2vkn4paXkNfQw8d98j6dwli5dLGs1uj2ryj6XvcnobCO4+5u7vZ7cnJH0+zXitr12ir76oI+y3STo55f4pDdZ87y7pN2b2npmN1N1MB7e6+5g0+ccj6Zaa+7lU4TTe/XTJNOMD89p1M/15WXWEvdNUUoM0/nePuy+R9ICkx7O3q5ieaU3j3S8dphkfCN1Of15WHWE/Jen2KffnS/qkhj46cvdPsutxSds1eFNRn/58Bt3serzmfv5hkKbx7jTNuAbgtatz+vM6wv6upIVm9iUzmyFphaSdNfRxGTO7KTtwIjO7SdI3NXhTUe+UtDq7vVrSjhp7ucigTOOdN824an7tap/+3N37fpH0oCaPyP9R0g/q6CGnrzsk7c8uH9bdm6RXNfm27v81+Y7oO5JulrRb0kfZ9ZwB6u2/JB2Q9IEmgzW3pt7+VZMfDT+QtC+7PFj3a5foqy+vG1+XBYLgG3RAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EMTfAa5yOtysgto/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(some_digit_image,cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[36000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立测试集和训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将X 进行拆分，拆分成训练集和测试集，60000个训练集，10000个测试集，该拆分方法过于一刀切，需要进行交叉洗牌\n",
    "\n",
    "X_train,X_test,y_train,y_test = X[:60000],X[60000:],y[:60000],y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([25753, 56981, 49193, ..., 59562, 27380, 21699])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将数据集合交叉洗牌，交叉验证时，每个子集合数据分布均匀，有些机器学习算法对训练实例的顺序敏感\n",
    "import numpy as np \n",
    "shuffle_index = np.random.permutation(60000)    # 对60000个下标进行随机的排列\n",
    "shuffle_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把训练集X进行重新排列\n",
    "X_train,y_train = X_train[shuffle_index],y_train[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=uint8)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练一个二元分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 识别数字5，二元分类是5和非5\n",
    "\n",
    "y_train_5 = (y_train == 5)\n",
    "y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       ...,\n",
       "       [False, False, False, ...,  True, False,  True],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_5.reshape(20,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_5 = (y_test == 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SGD 梯度下降分类器，适合非常大的数据集，独立处理训练集数据，一次一个，适合在线学习。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(random_state = 42)\n",
    "sgd_clf.fit(X_train,y_train_5)         # 先训练一个二分类\n",
    "\n",
    "sgd_clf.predict([some_digit])          # 预测36000那条数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 性能考核\n",
    "### 使用交叉验证测量精度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9669 , 0.95885, 0.9602 ])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 评估分类器比评估回归器要困难的多\n",
    "# scoring='accuracy'为精度评估，y_train_5判断照片是5，该分类器是单维度的，不是多维度的\n",
    "# 3个折叠，正确率达到95%以上\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf,X_train,y_train_5,cv=3,scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把每张图分类成 非5\n",
    "\n",
    "from sklearn.base import BaseEstimator\n",
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self,X,y=None):\n",
    "        pass\n",
    "    def predict(self,X):\n",
    "        return np.zeros((len(X),1),dtype = bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False],\n",
       "       [False],\n",
       "       [False],\n",
       "       ...,\n",
       "       [False],\n",
       "       [False],\n",
       "       [False]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.zeros((len(X),1),dtype = bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9062 , 0.9103 , 0.91245])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "never_5_clf = Never5Classifier()\n",
    "cross_val_score(never_5_clf,X_train,y_train_5,cv=3,scoring='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   * 准确率超过90%，因为5的图像大约只有10%，你猜一张图不是5, 90%的时间你都是正确的\n",
    "   * 这说明准确率无法成为分类器的首要指标，特别是当你处理偏科数据集，某些类比其他类更为频繁"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 混淆矩阵\n",
    "#### 多维度评估分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 评估分类器性能的更好办法是混淆矩阵\n",
    "# A类别实例被分为B类别次数\n",
    "# 想要知道分类器将数字3和数字5混淆多少次，通过混淆矩阵的5行3列\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "y_train_pred = cross_val_predict(sgd_clf,X_train,y_train_5,cv=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### cross_val_predict与cross_val_score 相比：\n",
    "      同样执行交叉矩阵\n",
    "      返回的不是评估分数，是每个折叠的预测\n",
    "      每一个实例在模型预测时使用的数据，在训练期间从未见过，为干净数据\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53645,   934],\n",
       "       [ 1347,  4074]], dtype=int64)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_5,y_train_pred)      # 将实际类别与预测类别交给混淆矩阵，二元分类器得到2*2的矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 查看矩阵：\n",
    "     行表示实际类别，列表示预测类别\n",
    "     右对角线为真分类，左对角线为假分类\n",
    "     这种衡量方式太复杂，可以用更简单的指标"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正类预测的准确率 被称为分类器的精度\n",
    "    精度 = TP/TP+FP\n",
    "    TP是真正类的数量，FP是假正类的数量\n",
    "    \n",
    "    召回率 = TP/TP+FN\n",
    "    检测实际正类的比例，FN是假负类的数量\n",
    "\n",
    "\n",
    "![jupyter](./zhaohui.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "### 精度和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8134984025559105"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 导入精度类和召回分数类\n",
    "from sklearn.metrics import precision_score,recall_score\n",
    "\n",
    "precision_score(y_train_5,y_train_pred)    # 实际标签与预测标签的精度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7515218594355285"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5,y_train_pred)       # 实际标签与预测标签的召回分数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 说明检测一张图的时候，检测是5是准确的概率79%，在所有数据中所有的5被检测出来的概率是78%\n",
    "### 精度和召回率合成单一指标，成为F1分数，称为谐波平均值\n",
    "### 谐波平均值会给予较低值更高的权重，只有召回率和精度都很高时，才能获得较高的F1分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7812829609742065"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5,y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 最终的f1分数为78.8%\n",
    "\n",
    "# f1分数对那些具有相近的精度和召回率 的分类器更有利，但不一定符合你的预期\n",
    "# 有时候更关心精度，有时候更关心召回率\n",
    "# 训练一个分类器检测儿童可以放心观看的视频，你可能要求拦截了很多好的视频，低召回率，保留下来的都是安全的视频，高精度。宁可错杀一千也不放过一个\n",
    "# 不能同时增加精度并增加召回率，反之亦然"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 精度/召回率权衡"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![jupyter](./quanheng.jpg)\n",
    "\n",
    "   * SGDClassfier 对每个实例基于决策函数计算一个分数，大于阙值为正类，小于为负类\n",
    "   * 提高阀值，向右移动，精度提高，召回率降低\n",
    "   * 反之阀值降低，向左移动，召回率提高，精度降低\n",
    "   * sklearn 不可以直接设置阀值，但可以访问决策分数\n",
    "   * SGDClassfier 默认阙值为0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([171.80377421])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 设置阀值\n",
    "# 返回决策值decision_function,决策值大于阀值则预测正类 返回true\n",
    "\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0          # SGDClassfier 默认阀值为0\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提高阀值可以降低召回率，提高精度，将阀值提到20000，就会错过这个图\n",
    "\n",
    "threshold = 20000\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   * 如何决定使用什么阀值：根据业务需求走，最终要求得到的精度和召回要求，针对性的调节阀值\n",
    "   * 返回决策值，而不是预测结果（method='decision_function'）\n",
    "   * 对整个训练集进行交叉预测，并获取决策值，通过多次折叠验证得到更加稳定的分数\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 返回整个训练集的所有实例的决策值列表，用了method='decision_function'方法\n",
    "y_scores = cross_val_predict(sgd_clf,X_train,y_train_5,cv=3,method='decision_function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 有了y_scores， 可以计算所有可能的阀值的精度和召回率\n",
    "y_scores.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 根据得到的决策值列表，可以得到实例的精度，召回和对应的阀值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# precision_recall_curve类设置两个参数 分别为训练集实例的标签值 训练集实例的决策值列表，可以返回实例的精度，召回和对应的阀值\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions,recalls,thresholds = precision_recall_curve(y_train_5,y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用matplotlib绘制精度和召回相对于阀值的函数图\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEPCAYAAABx8azBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3gVVfrA8e+bSgIBQgg1dJEWeuiIqCAIimIFdS27LmvB3+6KrAVXEOva3bUgNgQ7KogNxIaKIoSO9N4hdEJC6vn9ce5N46bBvZl7b97P88wzc2fOzLyTucmbOTNzjhhjUEoppZRzQpwOQCmllKrsNBkrpZRSDtNkrJRSSjlMk7FSSinlME3GSimllMM0GSullFIOKzUZi8ibIrJfRFYVs1xE5L8islFEVohIF++HqZRSSgWvslwZTwEGl7D8IqClaxgFvHLmYSmllFKVR6nJ2BjzE3CohCKXAlONtQCoKSL1vRWgUkopFezCvLCNhsCOAp93uubtKVpQREZhr54hiq7U9MLeld8JkRBExA4IoRIKAoLklcmb9jBfRAgR+3+iexvu7YaGhBJCSN78EAnJ21+ohBIeGk6ohOZtwz1ddFtKVZRt2+DAAc/LoqOhTZv8z4sXF14eFQVt2/ouNlWxFi9efMAYE+9pmTeSsae/bh7b2DTGTAYmAzRp28SMmzau4LIiGzBF163Q5f4QQ9HlpZUxxpCenU5Obg4GQ67JJdfkYkyBadd897zs3GzSstPIyc0hx+SQk5tDrsklx+SQnZvNofRDZOZkFlqeY3Ly1j2ecZzs3Oy8ISs3i1xyC8WYRdYpx+Gk6PBoIkIjCA8JJzw0nJiIGBrENCAuOo7o8GiqR1SnUY1GtK7dmlpRtYgKi6J+TH1qR9cmPCQ8L7krVRYLFsDGjZ6X1aoFQ4bkf37nncLLY2Nh6FDfxaYqlohsK26ZN5LxTqBRgc8JwO7SVoqPjmdU11Fe2L3yJzm5OZzIOkFWThZZuVmczD7JicwTpf5z4J6fnZtNWlZa3j8DBf8JOHzyMCcyT+Ql/cycTI5nHCcjJ4OMnAwOpx9mb+peUjNTycjJIDUzlfSsdLsd1/bc20/LSisU97qD68p0fIIQERpBbFQsVcOr0rB6Q85reh7NY5vTNr4tnep1IizEG79WKlj07GmHsrj+et/GovyXN/5qzAJGi8gHQA/gqDHmlCpqVTmEhoRSPbK602EUK9fkciLzBFm5WWTl2IS+78Q+DqUf4lD6IU5mn+RQ+iE2HdrEmgNr8pL6nuN7OJR+iByTQ0ZOBntT9wKw6fAmftr2U97246Pj6dekH+c2OZekBknUqVqH5rHN9Wq6GBs3wpIl0Ls3JCQUXpaSAqtWQUaGTWY1a8K+fbBmDfToYatw/Vl2NvTpA1WrwvffOx2N8ndSWq9NIvI+0B+oDewDxgPhAMaYSWL/yryIfeI6DbjZGJNc2o6TkpJMcnKpxZTyKzm5OaRnp3P05FFSM1NZtHsRS/csZfORzSzfu5wtR7acsk6DmAaM7jaake1H0rRm04oP2iGpqTBpEowdC5dfDrfdBgMGwM8/23FmZn7ZkSPhwQehdWuYPh1+/BEmT7YJzS09HaZNg1GjIDwcLrwQ7roLzj+/8H43bICGDe39WCelpkJMjI3jxAlnY1H+QUQWG2OSPC5zqgtFTcYq2BhjWLZ3Gcm7k/l2y7dsPmwTdFZu/j3zC1tcyDXtrmFA8wE0iGkQtFXanioCxo6FJ5+0yfS11wovi4yE//s/u3zYMPj888LLY2Lg2DF7T/WZZ2D5cnD/6br0UpgwATp1skn8vPPs1eirr8K113qOpSIcPAi1a9v7vodKeh9FVRqajJVySHZuNq8veZ0v1n/B1xu/JtfkP9xWLaIaTw98movPvpiG1Rs6GOWZS02FP/0JbrgB6tWz1c4F1ahhq2q7dIFff7XVt2DLv/124bJLlsCnn9rE27w5fPIJVK8OoaH5ZVJS4JVX4PHH4eRJm3BfecUm7euuyy83eDA88QR07Oib4y7J7t32Cr1ePdijN+4UmoyV8gu7ju1i+urpfLLmE37Z/kve/BAJYfBZg7mnzz30a9LPwQjLzxjo1i3/lZxbbrFXve6r0ZwcCPFho7t79tiEPGkSfPQRXHIJNGkCrVrZpH7kiC338MNw332FE7qvbdli/5lo0gS2bq24/Sr/FbDJ+OjRoxw4cIDMgjeXVKUQERFB7dq1qVGjhtOh+ER2bjaTF09m1rpZzN08N++K+ep2VzOm1xi6N+zucISle+wxGDeu8Ly//x2ef77iY9m82V49165t7zOHhcHevXDVVfDLL9C/v636rlat8Hq5ufafhX/9y/4D8fjj3vvnYe1a+w7x2WfDurI9rK+CXEAm45MnT7J9+3YSEhKIiorSp1ErEWMM6enp7Ny5k8aNG1OlShWnQ/KpHUd38OLCF3nmt2fIMTkA3NTpJl4Y/ILfPpnuvh9aUGamfbDK3/z6q60ed3+N9u+3sQ8dahPvBx/YRA72n4tHHvHOfpcvt/ex27eHFSu8s00V2EpKxn7ba1NKSgrx8fFER0drIq5kRITo6Ghq165NSkqK0+H4XKMajfjPwP+w4c4NjOoyivCQcKYsm0L317qzfO9yp8MrZPlye483Lg6++cbO++MPW13tj4kY7P1rdyLetAm6drXJd/Zs+Oqrwk9sF71yPhM1asBNN9kH0pQqjd8m45MnT1LNm78ZKuDExMRw8uRJp8OoMM1im/HqJa/y+y2/07p2a9YdXEeXyV2YOG8iObk5TofHuHH2Su+mm+z0wIE2CQdSc42bN8POnfmvGjVtWrha+t57vbevpk3hrbe8d6WtgpvfJuPs7GzCwoLztQ9VNmFhYWQXvGypJDrX78wvN//Cnzr8iVyTy/gfx9PnzT6knHCmliA93d5Pfeyx/Hm9ejkSyhkbONBe3Re0bNmp5Y4fr5h4lHLz22QMaPV0JVeZz39cdBxTh0/loys/IiYiht93/c6fZvzJYxvqvmTMqY1n/PEHXHxxhYbhVR062IZHWraEp5+GXbvyl2Vmwpgx0KIFzJx5Zvs5csQm/h07Si+rlF8nY6Uqu6vaXcWyW5cRERrBnE1z+POsP1doQjam8BXxiROBVS1dnL59Yf16uOIK2zDIggWwcqWtsl650r7HPHw4TJ3qeX1jbI1BSb7/3lbr//3v3o9fBR9Nxkr5ueaxzZl62VRCJIQpy6Zw9zd3V8h+s7JsO9B/+Ut+i1dONzHpKz16QGKifSXq66/hoYfs/Icftu9KFzVmjH2IbfPm4reZkWHHkZHej1cFH03GFWjKlCn5/fyKEBMTQ8eOHXnxxRcr9N7ohAkTfFYFLCJMmDDBJ9uuzK5JvIZ3L38XQXh2wbOFGg3xlYgI27TkkSO2areyCA2F+++HZs1sRxaPPnpqmTlz7JVx0S4PC9JkrMpDk7EDpk+fzm+//cYnn3xC9+7dufPOO5k4cWKF7f+WW27ht99+88m2f/vtN2655RafbLuyG5E4gr91/RsAV3x0BQfTDvpsX+4kk5Ji23mubMLC8hsv8VRV7e7q0N3ClyeajFV5aDJ2QKdOnejZsycXXnghr732Gv379+f5YpotMsZ4vQWyhIQEepa1g9Vy6tmzJwlF+8JTXvPYBY+RWCeR/Sf28485//DJ/ePcXNvOtFvDwG42+7QNGWLfR960Cb74ovCyNm3seP364tfXZKzKQ5OxH+jWrRvHjx9n//79NG3alOuvv54333yT1q1bExERwZdffglAWloa99xzD82aNSMiIoJmzZrx6KOPkpubW2h7KSkp3H777TRq1IjIyEgaNWrEn/70JzJcfx08VVO/8MILtGnThqioKGJjY0lKSmLGjBl5y40xPPfcc7Rq1YqIiAjq16/P6NGjOXbsWKHtFK2mdu9rw4YNDB06lGrVqtGkSRMmTpx4StyqdLFRsbx92dtEhkbyzop3fFJdXbD9Zh9VoASEsDB4/XWYMaNwN4179sCzz9rp1NTi19dkrMoj4JKxSPHD5Mn55SZPLrlsQV27Fl9u1Kj8cu7G8L1ty5YthIaG5jVy8sMPP/Dss88yfvx4Zs+eTYcOHcjOzmbQoEG8/vrr/P3vf+frr7/mlltu4eGHH2bs2LF52zp8+DC9e/fmww8/5K677uKrr77iySefJCsrq9gr7HfffZcxY8YwcuRIvvrqK959912uvPJKDhXo923cuHHcddddDBw4kM8//5x//etfTJkyhaFDh5YpqQ4fPpzzzz+fmTNnctlllzF+/HjeLtpdjyqTLvW7MKbXGADGfDPGqw2CvPde4c8+qkAJGFddBZddVvjBtdWr7atRAGlpxa+ryViVizHGkaFr166mJKtXr/Y43z7T6Xl49dX8cq++WnLZgrp0Kb7cX/+aXy45ucSQS/XWW28ZwKxdu9ZkZWWZQ4cOmUmTJpmQkBBz6aWXGmOMadKkiYmKijJ79uwptO7UqVMNYObNm1do/iOPPGLCw8PNvn37jDHG/Pvf/zYhISFmyZIlxcYxfvx4Q4Efwh133GE6d+5cbPmDBw+ayMhIc+ONNxaaP23aNAOYzz77LG8eYMaPH3/Kvt58881C6yYmJpqBAwcWu0+34r4Hld3BtIOm+uPVDRMwt31xm9e2u2GD/d5HR3ttk0Fj1y5jjhwxZsqU/L8P7dqVXP7XX43ZurXiYlT+DUg2xeTEgLsyLinFFryKHTWq5LIFLV5cfLmCV9tdu3rnGFq3bk14eDi1atXi9ttv57rrruPNN9/MW96zZ0/q1atXaJ3Zs2fTpEkTevfuTXZ2dt5w4YUXkpWVxYIFCwD45ptv6NatG507dy5zPN26dWPZsmXceeedfPvtt6QV+Xd/wYIFZGRkcL37qRWXESNGEBYWxrx580rdx9ChQwt9TkxMZPv27WWOURVWK6oWH175IQCvJL/CD1t+8Mp24+Lgyy+hyN2HSs0YOPdce+/8/PPt615uXboUv16DBralsiZNfB+jCnwBl4yDwYwZM1i0aBFr167lxIkTTJ06lVq1auUtr1+//inr7N+/n23bthEeHl5o6N7ddrV38ODBvHF5H6C64YYbeOWVV/j9998ZNGgQtWrV4vLLL2erqxNWd3V10bjCwsKIi4srVJ1dnILHBxAZGVmp2p32hcFnDebBfg8C8NC8h87oYa6tW20nCZ99Zh9cqsh+f/2diO0co0UL20fyf/5j57/4YvGNgihVXpqMHZCYmEhSUhKtWrXy2D2gp3eA4+LiaNasGYsWLfI4XHLJJQDUrl2bXQXb9ysDEeFvf/sbCxcu5MCBA7z99tssXLiQa665BshPpHv37i20XnZ2NgcPHiQuLq5c+1Pec3u324kKi2Letnk8+MODp72dZs1s61qffurF4IJI06YwbVrheaX9z/vOOzB6tO3CUanSaDIOEIMHD2bHjh1Uq1aNpKSkU4bars5lL7zwQhYuXMjyoq3hl1FsbCzXXHMNV199NatWrQJstXlkZCQffPBBobIffvgh2dnZnHvuuWd2cOq01a1Wl9cueQ2AR35+hN92lP/x58OH86f79vVWZMGnVy8oePeoUSPYt8+22OXJt9/CSy/B2rUVE58KbNotUoC47rrreOutt7jgggsYM2YMHTt2JDMzk02bNjFr1ixmzpxJdHQ0//znP3nvvfcYMGAADzzwAO3bt+fAgQN89tlnTJo0iZiYmFO2PWrUKGJiYujVqxd16tRh/fr1TJs2jQsvvBCwV8Z33XUXjz/+OFWrVmXIkCGsWbOGBx54gL59+55yP1hVrOs6XMfiPYt5bsFzPPDDA3x3w3flWn/AgPzpAg/mKw8SE8FdQRQebhNyeLh93al69cJl9WlqVR6ajANEeHg4c+bM4YknnmDy5Mls2bKFqlWr0qJFC4YOHUpERAQANWvWZP78+TzwwAM88cQTHDx4kLp163L++efnlSmqT58+vPXWW0ybNo2jR4/SoEEDrr/+eh5yN9ALPProo8THxzNp0iRefvll4uLiuOGGG3j88ccJCdEKFqf9u9+/eW3Ja3y/5XsW7lpI94bdy7zukiV23K3bqa/9qcLat7dXvI8+ahNz794wbx588gncfHPhspqMVbkU95i1r4fTfbVJVS76PSi7f3z9D8MEzBUfXlHmdb77Lv/dgR07fBhckPj0U2OuvtqOjTHmlVfsz27YsFPLDhlil82aVbExKv9FML3apJTybGyfsYRKKJ+t+4wDaQfKtE6DBvnT2opp6YYPhw8/tGOASy+142++sQ/AFeS+MvbwjKZSp9BkrFSQaBDTgPOanUd2bjaP/fxY6SsArVrB9OmFH+JSZVe/vm2l7ORJmD278DKtplbloclYqSDy737/BuDlRS+zL3VfiWWXLYOjR+HKK6FmzYqILjhddpkdz5lTeH7z5tCuHdSoUfExqcCjyVipINKvST8uOusiMnIy+N/C/5VYtnNniI2FlSsrKLgg1bOn7fvZfSXs9vbbsGoVdOzoTFwqsGgyVirIuDuRmLZiGlk5WR7LbNqUP924cUVEFbz69LG9N2m/J+pMaDJWKsic1+w8WsS2YPvR7Xy/5XuPZUaOzJ/WatQzExZm3zUu6gxaJ1WVkCZjpYJMiIRwY8cbAXhz2ZunLD9+HBYtstPdy/46sipFbm7hLhUTEuzDW+VsnVZVUpqMlQpCN3S8AYAv1n9BWlbhXrheeil/eu7ciowqeE2dau+/33pr/ryTJyEz095PVqo0moyVCkJNajahe8PupGWlMXdT4Yx73312PGLEqU04qtPjvm/8/vu2ByzQV5tU+WgyVipIXXTWRYC9Oi7ol18gKQkeK9uryKoMWrSAa6+F7Gx44gk7T5OxKg9NxhVoypQpiEjeEBERQYsWLbj//vsd79u3adOm3HTTTXmf3bG6+zRWgWd4a9tM1OfrPycnNydvfp8+9p5xs2ZORRacxo2zbXu/9RZs324TM2g1tSobTcYOmD59Or/99htffvklgwYN4vHHH2esdpejvKxD3Q60iG3BvhP7mLdtHgAHDsC2bQ4HFqRat4arrrL3iR95xM6LiNDON1TZlCkZi8hgEVknIhtF5F4PyxuLyA8islREVojIEO+HGjw6depEz549GThwIC+//DIDBgzgjTfeIDc31+nQVBAREa5tfy0A7618D4D4eGjaFHbudDCwIDZunB2/ZruY1ipqVWalJmMRCQVeAi4C2gIjRaRtkWIPAB8ZYzoDI4CXvR1oMOvSpQvp6ekcOJDfuP+WLVu47rrriI+PJzIykk6dOjFjxoxT1l2+fDnDhw8nLi6OqKgoWrVqxeOPP563/JtvvmHIkCHUr1+f6OhoEhMTeeaZZ8jJyTllWyr4jEy0LxR/vPpjDhzJvxVSu7ZTEQW3Dh1s5xGJiXDHHfD0005HpAJFWfoz7g5sNMZsBhCRD4BLgdUFyhjA/VxmDWC3N4N0k4f8o77HjPfu2/xbt26lRo0axMXFAbBjxw569OhBnTp1eO6554iPj+fDDz/kiiuuYObMmQwbNgyAhQsX0r9/f8466yyee+45EhIS2LBhAytWrMjb9ubNm7ngggu48847qVKlCsnJyUyYMIGUlBSecD9pooJWm/g2dK7XmaV7l3LnC18D9j6y9iTkO1OnQkyMVk+r8ilLMm4I7CjweSfQo0iZCcA3InInUBUY4GlDIjIKGAXQuBK3wZeTk0N2djbHjx9nxowZfPLJJzz//POEhoYCMGHCBIwxzJs3Ly9BDxo0iB07dvDggw/mJeO7776buLg4FixYQHR0NADnn39+oX3dWuDFR2MM55xzDpmZmTz99NM89thjhIToYwPB7qq2V7F071LW5czBnYyV7+jrYup0lCUZe/r/ruil4UhgijHmGRHpBUwTkURjTKGboMaYycBkgKSkpHJfXnr7itQprVu3LvT59ttvZ/To0XmfZ8+ezZAhQ6hRowbZ7kcysQl57NixHDt2jLCwMObPn8/YsWPzErEne/bsYcKECcyePZvdu3cX2t7+/fupV6+eF49M+aP+TfsDsC7DPsT18ccOBlNJLFsGb7xhn1wfMcLpaFQgKEsy3gk0KvA5gVOrof8CDAYwxvwmIlWA2sB+bwQZbGbMmEFCQgIpKSk8++yzvPzyy/To0YMbbrCtJu3fv5+pU6cydepUj+sfPHiQiIgIcnNzSSihR/jc3FyGDRvG7t27mTBhAq1btyYqKoqZM2fy6KOPOv46laoYSQ2SiA6PJi16LcTspm/fBk6HFPS6dLFtU7/4oiZjVTZlScaLgJYi0gzYhX1A69oiZbYDFwBTRKQNUAVI8WagwSQxMZGzzjoLsNXKHTp0YOzYsVxxxRVUrVqVuLg4zjnnHO655x6P6zdo0ICcnBxCQkLYVULDt5s2bSI5OZlp06Zx/fXX583//PPPvXtAyq+Fh4bTNfZ8ft7/BfF9vqRu3b86HVLQCwuDLM8dZinlUak3DI0x2cBoYA6wBvvU9B8iMlFEhrmKjQH+KiLLgfeBm4zRPkvKIjIykqeeeor9+/fz8sv2IfTBgwezYsUK2rVrR1JS0ilDZGQk0dHR9O3bl3feeYf09HSP205ztVofXqBLmaysLN59913fH5jyK1d1vASA5oO+dDiSysFTL05KlaQsV8YYY74Cvioy78EC06uBPt4NrfIYNmwY3bp14+mnn2b06NFMnDiR7t27069fP0aPHk3Tpk05fPgwq1atYvPmzbz5pu2J5+mnn+bcc8+lV69ejBkzhoSEBDZv3syyZcv43//+R5s2bWjSpAnjxo0jNDSU8PBwnnvuOYePVjnhsnYX8X9z4Y/078jMySQiVJuF8qWwMv1lVSqfPkrrJx555BH279/PpEmTaNy4McnJyXTs2JH777+fgQMHcttttzFv3rxCT0t369aN+fPn06hRI+68806GDBnCU089lXcfOSIigpkzZ1KvXj1uuOEG7rjjDvr168e9957SbosKcsd2NqJtXCKpmanM2zrP6XCCniZjVV7iVG1yUlKSSU5OLnb5mjVraNOmTQVGpPyRfg/O3PjxMHEiMOAe6PskY3qN4ekLtTUKX6pbF/a7Hl/VG3bKTUQWG2OSPC3TK2OlgtyyZa6JDUMBmLVulnPBVBLaOYQqL03GSgW5tWvteNIDvYgKi2LDoQ3sP6FvHfrSvHl20E7PVFlpMlYqiBkD69fb6d49wunbuC8A327+1sGogl/z5tCvHzRp4nQkKlBoMlYqiO3blz/drh0MbD4QgG82feNQREopT/w6GeurypWbnv8z963rArhzZwgJIe/KeOnepQ5GpZQqym+TcXh4eLGNWajKIT09vVCDJar8OnSw1aVtXZ2edqrXifCQcFbuW8mBtAMlr6yUqjB+m4zr1KnDrl27SEtL0yukSsYYQ1paGrt27aJOnTpOhxPQOnSAt9+2A0BUeBT9m/bHYPS+sVJ+xG9fTa/u6ods9+7dZGkjr5VOeHg4devWzfseqNPXtGnhz+c1PY+5m+cyZ9McRiRqLwZK+QO/TcZgE7L+MVbq9Jw4AZ9+ansQatcuf/6FLS7k/u/v15a4lPIjfltNrZQ6MytWwA03wHXXFZ7foW4HIkMj2XJkC4fTDzsTnFKqEE3GSgWpUaPsuGjlUnhoOB3rdQTg5+0/V3BUSilPNBkrFaRWrbLjunVPXeZ+3/j7Ld9XYERKqeJoMlYqSNWoYccPP3zqMncynrdN7xsr5Q80GSsVhA4fhqNHISoKWrY8dXnXBl0JlVBW7ltJamZqxQeolCpEk7FSQWjFCjtOTITQ0FOXV4uoRuf6nckxOSzYuaBig1NKnUKTsVJBKD0d2rSxrzUVp1/jfgD8tO2nCopKKVUcTcZKBaHBg2H1anjlleLLnNPkHECfqFbKH2gyViqIiRS/zN1pxIKdC8jIzqigiJRSnmgyVirI5OTA3r2ll6sdXZu28W05mX2SxXsW+z4wpVSxNBkrFWTWroX69aF799LL6n1jpfyDJmOlgszy5XbcsGHpZfs1sclY7xsr5SxNxkoFGXcy7tix9LJ9GvcB4Pedv/swIqVUaTQZKxVkypOMG1VvRM0qNTmYfpDtR7f7NjClVLE0GSsVZMqTjEWE3o16A2jjH0o5SJOxUkFk3z77JHVMDDRtWrZ1utbvCsD87fN9F5hSqkSajJUKIsnJdty5M4SU8bf7orMuAmD2ptk+ikopVZowpwNQSnlP//4wbx4YU/Z1utTvQlhIGBsObuB4xnFiImN8Fp9SyjO9MlYqiFStCv36wbnnln2dyLBIOtTtgMGwdO9S3wWnlCqWJmOlFJ3qdgJgxb4VDkeiVOWkyVipIHHkiG2L+uKLy79u2/i2APyx/w8vR6WUKgtNxkoFiZ9cLVp++WX51+1S3/a1mLwn2YsRKaXKSpOxUkFiu6vNjpo1y79u1wZdEYTle5drD05KOaBMyVhEBovIOhHZKCL3FlPmahFZLSJ/iMh73g1TKVUad2Mf48eXf93qkdVpXbs1WblZet9YKQeUmoxFJBR4CbgIaAuMFJG2Rcq0BO4D+hhj2gH/8EGsSqkSrFplx2VpecuTbg27AbBw10IvRaSUKquyXBl3BzYaYzYbYzKBD4BLi5T5K/CSMeYwgDFmv3fDVEqVxBj4w/XsVWLi6W2jWwObjBftXuSlqJRSZVWWZNwQ2FHg807XvILOBs4WkfkiskBEBnvakIiMEpFkEUlOSUk5vYiVUqfYuhWOH4e6dSE+/vS2oclYKeeUpQUu8TCvaPs+YUBLoD+QAPwsIonGmCOFVjJmMjAZICkpqRxtBCmlSvOXv9hGP05Xx3odCQsJY03KGm2JS6kKVpYr451AowKfE4DdHsp8ZozJMsZsAdZhk7NSqgI0awavvw4vvHD626gSViWvJa4le5Z4LzilVKnKkowXAS1FpJmIRAAjgFlFyswEzgMQkdrYauvN3gxUKeV7WlWtlDNKTcbGmGxgNDAHWAN8ZIz5Q0QmisgwV7E5wEERWQ38AIw1xhz0VdBKqcK+/ho2bixfBxGeaDJWyhll6rXJGPMV8FWReQ8WmDbAXa5BKVWBUlNhyBAID7fTERp8QPwAACAASURBVBGnv63uDbsD+nqTUhVNW+BSKsCtXGnHbdueWSIGaBPfhujwaLYe2UrKCX3jQamKoslYqQC3bJkdn25jHwWFhYTltVOtVdVKVRxNxkoFuP/+146bNPHO9no27AnA7zt/984GlVKl0mSsVIDbssWOzz7bO9vrmWCT8YJdC7yzQaVUqTQZKxXAsrMhxPVbfNFF3tlmj4QegL0yzsnN8c5GlVIl0mSsVADbvh1EoGlTiIvzzjYbxjQkoXoCRzOOsjpltXc2qpQqkSZjpQJY8+Zw7Bj8/LP3tikieVXVi/cs9t6GlVLF0mSsVIALDYWEBO9us0OdDgCs2r/KuxtWSnmkyVipAHamLW4Vp1O9TgAs3bvUNztQShWiyVipAJWTY7tM7NMHMjK8u+1uDW2zmAt3LSTX5Hp340qpU2gyVipArV0LKSmwaxdERnp32/Wq1aNu1bqkZqay8dBG725cKXUKTcZKBajFrmerunb1zfbdV8fanaJSvqfJWKkA5etk3LleZwCW7V3mmx0opfJoMlYqQFVUMtYrY6V8T5OxUgEoJweWuh509lUybl+3PaCvNylVETQZKxWA1q2DtDRo3Bhq1/bNPprHNqdmlZrsSd3DjqM7fLMTpRSgyVipgFS3Lrz5JjzwgO/2ESIhdG/YHdDuFJXyNU3GSgWguDi4+Wb46199ux+9b6xUxdBkrJQqVq+EXgDM3zHf4UiUCm6ajJUKMDk5cPfd8M47vmsO0613o96A7U4xMyfTtztTqhLTZKxUgFm/Hp55Bu6/33af6EvxVeNpXbs16dnpLN2j7VQr5SuajJUKMO73i5OSKmZ/5zQ+B4Cft3uxn0alVCGajJUKMMnJduyr94uL6tu4LwC/bP+lYnaoVCWkyVipAOPrlreKKpiMja9vUitVSWkyViqAVETLW0U1q9mMWlG1OJh+kN3Hd1fMTpWqZDQZKxVA1q+HEyegUSOIj6+YfYoI7eLbAbBy/8qK2alSlYwmY6UCyPHj0K0b9OlTsfvtWLcjACv2rajYHStVSYQ5HYBSquy6d4eFC33/fnFRHep2ALQlLqV8Ra+MlQpAvn6/uKieCT0BWLBzQcXuWKlKQpOxUgEiN9feM87Nrfh9t41vS0xEDNuObmPP8T0VH4BSQU6TsVIBYv16aNUK2rev+H2HhoTSrWE3AOZtm1fxASgV5DQZKxUg3O8Xt2zpzP7Pa3oeoI1/KOULmoyVChAV3dhHUe5mMX/d8aszASgVxDQZKxUgnnvOjp1Kxt0adiNUQlm+bzmpmanOBKFUkNJkrFQAyM7On66oDiKKig6PpnP9zuSaXBbuWuhMEEoFqTIlYxEZLCLrRGSjiNxbQrkrRcSIiEN/LpQKTqtX50/XqeNcHL0TbP/G87fPdy4IpYJQqclYREKBl4CLgLbASBFp66FcDPB/wO/eDlKpym7RIju+5hpn4+jVqBcAP2770dlAlAoyZWmBqzuw0RizGUBEPgAuBVYXKfcw8CRwt1cjVEpxww3QpQuEhjobx6AWgwiREH7a9hPpWelEhUc5G5BSQaIs1dQNgR0FPu90zcsjIp2BRsaYL0rakIiMEpFkEUlOSUkpd7BKVVbh4dC5M3To4GwcsVGxtK/TnuzcbJJ3JzsbjFJBpCzJ2FPDe3kt44pICPAcMKa0DRljJhtjkowxSfEV1eWMUsqreiXYqurfdv7mcCRKBY+yJOOdQKMCnxOAgp2axgCJwI8ishXoCczSh7iU8o45c6BHD3j5Zacjsdz3jTUZK+U9ZUnGi4CWItJMRCKAEcAs90JjzFFjTG1jTFNjTFNgATDMGKN1WEp5wbx5tqembducjsTq3cg+Uf3rjl/JNQ40lK1UECo1GRtjsoHRwBxgDfCRMeYPEZkoIsN8HaBSld1811tEffs6G4dbi9gWNKreiP0n9rN492Knw1EqKJTpPWNjzFfGmLONMS2MMY+65j1ojJnloWx/vSpWyjsyM+1VMUDv3s7G4iYiDGw+ENB2qpXyFm2BSyk/9vvvcPIktG0LcXFOR5OvR0IPQO8bK+UtmoyV8mPuKupzz3U2jqLOb3Y+AN9u/pbs3OxSSiulSqPJWCk/9qurgyR/qaJ2O6vWWbSIbcHhk4e1FyelvECTsVJ+7Kab4M47/efhrYKGtx4OwCerP3E4EqUCnyZjpfzY5ZfDf/8LTZs6HcmpLmp5EQALdi1wOBKlAp8mY6XUaelSvwsAy/cuJysny+FolApsmoyV8lNPPAHTpkFqqtOReFazSk3OqnUWGTkZrNy/0ulwlApomoyV8kNpaTB+PNx4o321yV/1a9wPgC/Wl9hHjFKqFJqMlfJDP/9sG/zo3Blq13Y6muINb2Mf4pq9cbbDkSgV2DQZK+WH5s6144EDnY2jNL0SeiEIybuTOZF5wulwlApYmoyV8kNffmnHgwY5G0dp4qLjSGqQRFZuFj9t+8npcJQKWJqMlfIzGzfC2rVQo4Z/vl9c1IDmAwDbGpdS6vRoMlbKz0yfbscXXwzh4c7GUhbupjHnbZvncCRKBS5Nxkr5mXr14JxzYMQIpyMpm14JvQgPCWfJniWkZvrpe1hK+TlNxkr5mZtvhp9+slfGgaBqRFXa122PwWg71UqdJk3GSqkz5u7f+KsNXzkciVKBSZOxUn7knXcgORmMcTqS8rmizRUATFsxTbtUVOo0aDJWyk8cOwa33ALdu8PevU5HUz5JDZI4O+5sDqUf4pftvzgdjlIBR5OxUn7ik08gI8M+vFW/vtPRlI+IMOzsYQB8uf5Lh6NRKvBoMlbKT7zxhh3feKOzcZyuwWcNBuDbLfq+sVLlpclYKT+wdi3Mnw/VqsHVVzsdzenp3ag3kaGRLNu7jL2pAVbPrpTDNBkr5QfeesuOr7nGJuRAFBUexcAW9qnqWetmORyNUoFFk7FSDsvKgrffttN//rOzsZypy1pdBsBn6z5zOBKlAosmY6UclpoKw4dDz57Qq5fT0ZyZi8++GEH4dvO3HM847nQ4SgUMTcZKOSw2Fl55BX79FUScjubM1K1Wl96NepOZk8nXG792OhylAoYmY6X8RKAnYrfL21wOaFW1UuWhyVgpB334ob1ffOKE05F4j/sVp7mb5pKVk+VwNEoFBk3GSjkkIwPGjIGbboIff3Q6Gu9pU7sNLWJbkJKWoh1HKFVGmoyVcsi778KuXdCuHVx0kdPReI+I5F0dz1g7w+FolAoMmoyVckBGBkycaKfvuQdCguw3cWTiSADeX/U+JtB6vVDKAUH2J0CpwPD667BtG7RtC9de63Q03te7UW8axjRk/4n9rDmwxulwlPJ7moyVqmBpafDII3Z64kQIDXU2Hl8QETrV6wTAwl0LHY5GKf+nyVipCjZliu0isWtXuPxyp6PxnWGtbC9Obyx9w+FIlPJ/moyVqmB//Su8+io89VTwvFvsycjEkVSLqMYv239hdcpqp8NRyq+VKRmLyGARWSciG0XkXg/L7xKR1SKyQkS+E5Em3g9VqeAQHg6jRsF55zkdiW/FRMZwXfvrAJi8eLLD0Sjl30pNxiISCrwEXAS0BUaKSNsixZYCScaYDsDHwJPeDlSpQLd3L+zc6XQUFetvXf8GwNvL3yY9K93haJTyX2W5Mu4ObDTGbDbGZAIfAJcWLGCM+cEYk+b6uABI8G6YSgW+O++E1q3h88+djqTidK7fmW4NunHk5BGmr57udDhK+a2yJOOGwI4Cn3e65hXnL4DHFuJFZJSIJItIckpKStmjVCrATZ8OH39spzt2dDaWiua+On518asOR6KU/ypLMvb0iInHt/hF5HogCXjK03JjzGRjTJIxJik+Pr7sUSoVwDZuhFtusdP/+Q80buxsPBVtROIIYiJi+HXHr6zct9LpcJTyS2VJxjuBRgU+JwC7ixYSkQHAOGCYMSbDO+EpFdjS0uCKK+DYMfsa0+23Ox1RxasaUZXrO1wP6INcShWnLMl4EdBSRJqJSAQwAphVsICIdAZexSbi/d4PU6nAk5MDnTvDihXQsiW89VZwv8pUkluTbgXgtSWvsePojlJKK1X5lJqMjTHZwGhgDrAG+MgY84eITBSRYa5iTwHVgOkiskxEZhWzOaUqjZUrYf16O/3JJ1C9urPxOKlD3Q4Mbz2cjJwM3lnxjtPhKOV3xKlG3JOSkkxycrIj+1aqIqSmwr33wqBBcMklTkfjvK83fM2Q94ZQr1o9Nv/fZqLCo5wOSakKJSKLjTFJnpZpC1xKedm6dXZcrRq8+KImYrfBZw2mS/0u7E3dq/eOlSpCk7FSXmIMPPyw7Z/400+djsb/iAjjzx0PwOO/PE5qZqrDESnlPzQZK+UFWVn29aUHH4TcXDhyxOmI/NMlZ19C94bd2XdiH1OXT3U6HKX8hiZjpc5QSgoMHgxvvglRUfZhrT//2emo/JOIcFvSbQB8tu4zh6NRyn9oMlbqDHz7rX196fvvoW5dOx4+3Omo/NvQlkMJlVC+2/wdWw5vcTocpfyCJmOlTlNmpu0Ocdcu6N0bFi+Gnj2djsr/xVeNZ2T7keSYHCYlT3I6HKX8giZjpcohK8u2pgUQEWEb8nj4YZg3DxqW1GK7KuTWrrYRkA/++IBck+twNEo5T5OxUmWQmwsffWSflL777vz5/fvDAw9AWJhjoQWkXo160aRGE7Yf3c6X6790OhylHKfJWKkS5OTAzJnQvTtccw1s2AC//AIZ2vr6GQmREO7odgcAryS/4nA0SjlPk7FSHhw+DE88Ac2b2weyFi+G+vXh1Vdh+XKIjHQ6wsB3c+ebiQyN5OuNX/P2sredDkcpR2kyVsrl+PHC0/fdB9u3w1lnwfPP264QR42C8HDnYgwmtaNrM/G8iQDcPfdu9qbudTgipZyjyVhVWocOweefw113QceO0L69bUULbJ/DDzwAX31lm7f8+98hOtrZeIPR2N5j6dekHwfSDnDzZzc7HY5SjtGOIlSlsmqVbS96/nw7XVDVqrBmDTRq5Hld5Ru7j++m5f9akpaVxidXf8LlbS53OiSlfEI7ilCVRnY2bNpkr2iffRZuvhkmFXiV9cgRe9931Sr7alLfvvYK+Lvv4MABTcROaBDTgIf6PwTAQ/Me0ledVKWkL2SogJGdbRPmvn22oY0BA2xCBVuN/MUX9h5vdnbh9Q4cgFvta6107gxPPQU9ekC3blClSsUeg/Lsjm538PyC51mxbwUvL3qZ0d1HOx2SUhVKk7GqcMeO2eH4cdvn7/Hj9vORI9CqFfTqZcstWQJjxti2n/fvt0m14F2VTZvs085gy2zebKcTEux2WrWCtm1t4nWrWrXwe8LKP0SFR/HC4Be4cvqV3PPtPQw+azBn1TrL6bCUqjCajEthjG3wwT3tTgbGgEjhJ2vT0k4t456uUiX/Ki4jwyahgmUKlq1Tx24bbJLJzPRctmpVqF3bLjt5ErZssbHm5OSP3UP79rZ/XbD3Rbdvt2Wys+2QlWXHNWrA0KH5+3vhhfwy7nIZGXa4+ur8xDl7Nrzyio3Dvdw95Obafbr17g1//OH553377fnbzM6GH3/MXyYC8fG2DegGDQpfAT/0EIwfbx+8itI+6wPSFW2vYGTiSN5f9T43zbyJH2/6kbAQ/ROlKgljjCNDRERX06iRMQkJxjRsaEyDBnaoX9+YKVNMnmnTjImPN6Z2bWPi4uxQq5YxsbF2yMnJL3vuucZUq2aHqlWNiY62Q1SUMbfdll9u6VJjwsONCQszJjTUDiEhxojYYeHC/LK33lo0BeYPHTuaQkSKL/vyy/nlXnut+HJgTHZ2ftmkpOLL/eUv+eWWLCl5mwWPadSo4st16pRfLje35G0WPKbJk0sum5GRX3bAAHuuW7UypmtXY/r3N2bYMGNuvNGYt97KL3fsmDHffWfM8uXG7N5tTFaWUUHuYNpBU+/peoYJmL989pfSV1AqgADJppic6Ni/nZmZsGOH52VpafnTJ0/aq8PiFKy2PHEi/4rT0/4KrpOVVbZtiuQPBT/DqU0gRkfbq0BP5QteQUdGQmzsqWXcQ8H9x8fbxiaKlg0JgVq18stFRUHr1nZZaKgdQkLypwu+ltOuHQwcmL8sPNweS1gYNGtW+Nj/+U+7HffysDB7hR8Zaa9w3QYOtC1VRUaeOlSpUvhnNXdu8T/7gmJi4Pzzy1ZWBYdaUbV4/4r3GfTOIN5Y+gaXtrqUS1pd4nRYSvmcY682deiQZD7/PPmURARQs6atggVIT89vjMFTkiuY1FJT85Nh0fJhYfmtJrmrZ4vbZsF5SqmK9/SvTzN27lhqRNbglz//QmKdRKdDUuqMlfRqk75nrJTyOzm5OVz98dV8uuZTOtbtSPKoZL1/rAKevmeslAoooSGhTL1sKo1rNGb5vuVM+HGC0yEp5VOajJVSfqlqRFXeHPYmAE/Of5LVKasdjkgp39FkrJTyWxc0v4BbOt9CVm4W1396PYfSDzkdklI+oclYKeXXnhz4JM1qNmPp3qUMfmcwB9MOOh2SUl6nyVgp5ddio2L5/sbvqVO1Dot2L6Ll/1ry7op3tQ1rFVQ0GSul/F7Tmk1Z8JcFnNf0PA6fPMz1M65n+IfDOZ5xvPSVlQoAmoyVUgGhWWwzvrvhO14e8jJVw6sya90sLph6AUdPHnU6NKXOmCZjpVTAEBFu63YbS/62hGY1m7Fo9yIGvTOITYc2OR2aUmdEk7FSKuCcHXc2393wHQ1jGvL7rt/pOKkjzy94npQTJbSdq5Qf02SslApIzWKbseK2FYxMHMmJrBP8c84/afJ8E26ceSPJu7V1PxVYNBkrpQJWrahavHfFe3x81cckNUgiPTudqcun0uP1Htz2xW3sTd3rdIhKlYm2Ta2UChprUtYwKXkSLy16iRyTQ3R4NFe3u5prE6/lguYXECJ6/aGcox1FKKUqldUpqxn3/Thmrp2ZN69WVC2GthxK70a96Vq/K0kNkhDtnk1VIE3GSqlKaeW+lby/6n2mrZjGzmM7Cy1rGNOQNvFt6JXQi0EtBtEjoYf2DKV86oyTsYgMBl4AQoHXjTFPFFkeCUwFugIHgWuMMVtL2qYmY6VURcnJzWHl/pX8uPVHkncnM3fzXPaf2F+oTLWIavRr0o9uDbqRUD2BetXq0SCmAc1jm1M9srpWcaszdkbJWERCgfXAQGAnsAgYaYxZXaDM7UAHY8ytIjICGG6Muaak7WoyVko5JdfksiZlDRsPbWTu5rl8sf4Lth3dVmz5EAmhRmQNYqNiia0SS80qNQtPu8ZhIWGESIjHQUSKXZZXhlPLxETG0KV+lwr86ShfOdNk3AuYYIwZ5Pp8H4Ax5vECZea4yvwmImHAXiDelLBxTcZKKX+y69guftj6A6tTVrM3dS97Uvew/eh2dhzdwfFM55rd7FSvE0v/ttSx/SvvKSkZl+UGSUNgR4HPO4EexZUxxmSLyFEgDjhQJJBRwCjXx1QRWVeG/Z+J2kVjCEDBcAygx+FPguEYIDiOo9RjWMYy5Fa/f9AsGM4F+P44mhS3oCzJ2NO3oOgVb1nKYIyZDEwuwz69QkSSi/svJFAEwzGAHoc/CYZjgOA4jmA4BtDj8IayPJGwE2hU4HMCsLu4Mq5q6hqA9gKulFJKlUFZkvEioKWINBORCGAEMKtImVnAja7pK4HvS7pfrJRSSql8pVZTu+4BjwbmYF9tetMY84eITASSjTGzgDeAaSKyEXtFPMKXQZdDhVWJ+1AwHAPocfiTYDgGCI7jCIZjAD2OM+ZYox9KKaWUsvQtdqWUUsphmoyVUkophwVUMhaRTiKyQESWiUiyiHR3zRcR+a+IbBSRFSLSpcA6N4rIBtdwY4H5XUVkpWud/4qrxXgRqSUic13l54pIrI+O5U4RWScif4jIkwXm3+eKaZ2IDCowf7Br3kYRubfA/GYi8rsr3g9dD9khIpGuzxtdy5v66DjuFhEjIrVdnwPqXIjIUyKy1hXrDBGpWWBZQJ2LsigudgfjaSQiP4jIGtfvwt9d8z2ee29+v3xwLKEislREvnB9Lvf3obzfOR8cQ00R+dj1O7FGRHoF6Ln4p+v7tEpE3heRKn5/PowxATMA3wAXuaaHAD8WmP4a+75zT+B31/xawGbXONY1HetathDo5Vrn6wLbfRK41zV9L/AfHxzHecC3QKTrcx3XuC2wHIgEmgGbsA/NhbqmmwMRrjJtXet8BIxwTU8CbnNN3w5Mck2PAD70wXE0wj7Ytw2oHaDn4kIgzDX9H/c+Au1clPFYi43dqQGoD3RxTcdgm95tW9y59+b3ywfHchfwHvDF6XwfTuc754NjeBu4xTUdAdQMtHOBbYRqCxBV4Dzc5O/nw7FfwtP8Ic/BdkIBMBJ4zzX9Kra9bHe5ddhf8pHAqwXmv+qaVx9YW2B+Xjn3uq7p+sA6HxzHR8AAD/PvA+4rcry9XMOcouVcX+gD5CeTvHLudV3TYa5y4uXj+BjoCGwlPxkH1LkocjzDgXcD8VyU8fg8xl7RcZQS42fYdvA9nntvfr+8HHcC8B1wPvDF6Xwfyvud88ExVMcmMSkyP9DOhbtFyFqun+8XwCB/Px8BVU0N/AN4SkR2AE9jfwjgucnOhqXM3+lhPkBdY8weANe4jpePAeBs4BxXlcg8Eenmml/e44gDjhhjsj0cR6EmSgF3E6VeISLDgF3GmOVFFgXauSjoz9j/1iGAzkU5FBe7X3BVD3YGfqf4c+/N75c3PQ/8C8h1fT6d70N5j83bmgMpwFuu6vbXRaQqAXYujDG7sPlhO7AH+/NdjJ+fD7/rvFNEvgXqeVg0DrgA+Kcx5hMRuRr7fvMAim+Os7zzvaaU4wjDVt/0BLoBH4lI8xLi8vRPU2nHccbHWMox3I+t4j1ltWL265fnwhjzmavMOCAbeNe9WjFxOXIuvMRf4jiFiFQDPgH+YYw5VsKtRH/8fl0M7DfGLBaR/u7ZJey3vLEW953ztjCgC3CnMeZ3EXkBWy1dHL87FwCue9qXYquWjwDTgYtK2LdfnA+/S8bGmAHFLRORqcDfXR+nA6+7potrsnMn0L/I/B9d8xM8lAfYJyL1jTF7RKQ+ULjTU+8cx23Ap8bWcSwUkVxsA+UlNT3qaf4BoKaIhLn+oytY3r2tnXKaTZQWdwwi0h77RV/u+qOZACwR+0BdQJ0L1/HcCFwMXOA6J5RwHBQz36fnwkvK0rRthRORcGwiftcY86lrdnHn3pvfL2/pAwwTkSFAFWx17/OU//tQ3u+ct+0Edhpjfnd9/hibjAPpXIC9QNtijEkBEJFPgd74+/nwdn29LwdgDdDfNX0BsNg1PZTCDxIsdM2vhb0HEusatgC1XMsWucq6HyQY4pr/FIUfVnjSB8dxKzDRNX02tspDgHYUfmBgM/ZhgTDXdDPyHxho51p/OoUfSrjdNX0HhR9K+MiH52Ur+feMA+1cDAZWY7v8LDg/IM9FKcdabOxODa5zPhV4vsh8j+fem98vHx1Pf/If4CrX9+F0vnM+iP9noJVreoLrPATUucD2KvgHEO3az9vAnf5+Phz7JTzNH3JfbN3/cux9pa6u+QK8hH3CbSWQVGCdPwMbXcPNBeYnAatc67xIfmtkcdgHMTa4xrV8cBwRwDuu/S8Bzi+wbJwrpnUUeNIQ++TieteycQXmN8c+objR9WVzP6FdxfV5o2t5cx+el63kJ+NAOxcbsf8MLXMNkwL5XJTheD3G7mA8fbFVfCsKnIMhxZ17b36/fHQ8/clPxuX+PpT3O+eD+DsBya7zMRObTAPuXAAPAWtd+5qGTah+fT60OUyllFLKYYH2NLVSSikVdDQZK6WUUg7TZKyUUko5TJOxUkop5TBNxkoppZTDNBkr5WVie7EqbdjqKjtFRHaWsskKISITXLF5pTEg9/bKUK6/a7/9vbFfpQKR37XApVQQ6FXk8wzsu/ETCszLqLBolFJ+T5OxUl5mjFlQ8LOIZAAHis4/UyISaYzRpK5UENBqaqX8gIh0FpGfRSTN1fn5rUWW3+Sqyu0nItNF5Ai2FTr38nNF5DsROS4iJ0RkjogkFtnGIBGZLyJHRSTV1Tn6gx7CaSYiX7rKbBORB0UkpMi2WonIDBE5IiLpIrJARAaX4TjjReQ9ETnmWncqts9cpSo1TcZKOa86tlP6d7C9zSwCXhGR8zyUfRfb1u+VuHrUEZGh2GYKU4HrgWuBGOBnEWnkKtMcmIVtuvQaYBjwLFDVwz5mAN8Dl2GbRHwIuNG9UEQaAL9g+7IeDVyN7R3nSxHx1DtOQZ9iO+W43xVHNvC/UtZRKuhpNbVSzovBNlr/A4CI/ITtnnIk8EORsh8bY/5VZN4LwDxjzKXuGSLyA7Yx+zHYfsC7YNtEv80Yc8xV7Pti4nnGGPOWa/pbETnfFYt73l3YNot7GWM2uvb3FbbDjUfJ7xO6EBEZiG2LeqQx5gPX7Dki8jWFe/NRqtLRK2OlnJfmTsQArvvAG4DGHsrOKPhBRFoCLYB3RSTMPQBpwG9AP1fRZUAW8IGIXCkidSjel0U+ryoSSz9ggTsRu2LOAd4HOolI9WK22wvIwXaXWNAHHsoqValoMlbKeYc9zMvA9iZT1J4in91J9Q1ssi04XIztcQdX4hyE/Z2fBuwVkd9F5FwP+yja13LRWGp5iANgL7Ynn1gPywDqA4eNMVlF5u8rprxSlYZWUysVWIq+t3vQNb4P+NZD+cy8Fe3V9w8iEgn0ASZi7/M2NcYcKEcMh4B6HubXc8VXNJm77QFiRSS8SEKuW459KxWUNBkrFdjWYR/KameMeaIsK7iqwb8XkWrAZ9hO0suTjOcB/3Al8a0AIhKKfSBrqTHmeDHr/YbtnP0KCldNjyjHvpUKSpqMlQpgxhgjIncAn4lIBPARNrHWBXoD240xz7peleoHfAXsAGpjr6Z3Y+8Jl8dzwE3AXBEZDxwDbgfOnq4ghgAAALNJREFUBoaWEOtcEfkFeFVEamPvi18DJBa3jlKVhd4zVirAGWO+wibaqsDrwBzgSWy18W+uYstdyx8HvgFexL4idb4xJr2c+9uNfSr6D+AV4GPsfeShxpjZpax+OfYfgseBD7EXBKPLs3+lgpEYU2rTsUoppZTyIb0yVkoppRymyVgppZRymCZjpZRSymGajJVSSimHaTJWSimlHKbJWCmllHKYJmOllFLKYZqMlVJKKYf9P1edJWLc7iFQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_recall_vs_thresholds(precisions,recalls,thresholds):\n",
    "    plt.plot(thresholds,precisions[:-1],\"b--\",label=\"Precisoin\",linewidth=2)\n",
    "    plt.plot(thresholds,recalls[:-1],\"g-\",label=\"Recall\",linewidth=2)\n",
    "    plt.xlabel(\"Threshold\",fontsize=16)\n",
    "    plt.legend(loc=\"upper left\",fontsize=16)\n",
    "    plt.ylim([0,1])\n",
    "\n",
    "plt.figure(figsize =(8,4))\n",
    "plot_precision_recall_vs_thresholds(precisions,recalls,thresholds)\n",
    "plt.xlim([-90000,90000])\n",
    "plt.show()\n",
    "\n",
    "# 由下图，可根据精度/召回率的要求，调节阀值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PR曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF5CAYAAACV7fNGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZgU1bnH8d87M+yLgoyoEQXUCEZFdNxwQ6OJckNwR/GqUSMJJkZuYuKOxvXGmGiMouEGJWo0QlS87rgRxOjVwQXFqFEEVyKKAjMg63v/OD2pnmGWnpnuql6+n+eZp053n656p5iHX62nzN0FAACKX1nSBQAAgHgQ+gAAlAhCHwCAEkHoAwBQIgh9AABKBKEPAECJIPQBACgRsYe+mfU1s2ea+byDmT1gZs+a2alx1gYAQDGLNfTNrJekP0nq1ky3MyXNcfd9JB1tZj1iKQ4AgCIX957+OkmjJS1rps9wSVNT7VmSqnJcEwAAJaEizoW5+zJJMrPmunWT9FGqvURS34YdzGyspLHh1Sa7bbppf/Xrl81KAQDIX3PmzPnM3Stb+71YQz9DNZK6SFoqqXvqdT3uPknSJEkyq/LRo6t1/fWx1ggAQGLMbGFbvpePV+/PkbRvqj1E0oLkSgEAoHgkuqdvZgdJ2sHdb0h7+0+SHjaz/STtIOn/EikOAIAik8ievrsPT02fahD4cveFkg6R9Kykg919XfwV1ldbm3QFAAC0Xz6e05e7f6zoCv7YrVolPfmk9Nhj0iOPSP/8pzRokLT//tLs2dL220sjRkjPPRd+PvpIuvhi6ZVXws+OO0p//rPU8HrFZcukt96S3nwz/NS1zzhDeu89af78MH35Zel735O+/FJauFB6443w/d13D8vdZBNpzz2lrbaSevWKffUAAAqUuXvSNbSLWZWfeWbmF/LV1EiLFknbblv/fXfp2Wel226Tpk0Lgdsezz8vffCBNGeO9NJL0muvSZ980r55Nmbs2LCc998PGw0rV0rXXCMtWSJ9+GFY7qhRUlmZNHCgtMUWUteuoZaPPw7TTz6R5s0LGxEbbRQ2Vr7+denTT8PP4sX12ytWhGUffrg0frz01VfSZ5+Fz+bPl8aMkfbYIywTAJB9ZjbH3Vt9S3tJhf4TT4Sgqq0Ne+Lf/a7UpYt0zz3S1VeHgK6zyy7Sd74TwuuKK8Ie9X77SZdeKlVWSsOGSXvvLc2cGcJ1112loUPDXntTOncOYbr99uHIwQsvSMuXS4MHh0AeODC8fvVVacCAEML9+klPPRX2+J9/Xpo7Vyovl9YlftIjc337Sv/6V/T6rrukb3xD2nJLjlQAQFsQ+s2Evrv0619L55xT//1+/aTevUPISiHMTztN+s//DKHUFkceKd13XzgEv8ce0m67hQ2CXXaRtt46e3u/jz0m/e1vITj79QsbCHffHTZCdtwxvP/yy2Gj5i9/kfbaK5yK2HzzsLefPu3WLRzlGDw4bADtvntYF5tuuuFPWVnYCBk/PmygDBkS+vbpI737rvTii637PSorwxGC8ePDRs9220VHFhYvDqcxuncPR2cGDw7rkiMIAEodod9M6J9/vnTVVeGwdWO/7lZbSeedJ518cgjJ9lq2TOrRY8Nz+qVi9Wrp88/DEZBOncIGweLF0g9/GI5WtNdJJ0kdOoSjBIsXR6cWXnghXHcxa5Z01lnSmWdK22wT/s1L9d8CQHEi9JsI/ZtuCofcKyqkO+4IQXHuuWEvWAoX4P3iF+E8N+JTUxP+DebODQHdqVM4OrH11mHvv7w8nIL5xjfCXn/66YH2aHiq4YILwhGgHjzhAUABIfQbCf25c6WqKmnNGmnKlLAnL0nr10v33ivttFM4v47C8LOfhb32OXPCv2tlZfSzYkX4bMmSsCH3+edtW8Y++4RTMh9+KP397+EiyJNPljbeODqi8MknYeNxzJjs/n4AkClC/3rpJz+RHn443DbXpUs4N/3yy+Gw8k03JV0p4vbxx+EIzvLl4TbMmpqwgfCvf4WLM++9NzvLOecc6Uc/Cqd1dtiBUwkAcq/kQ/+aa8Ih4joHHxyu1u/XL9zn3r17cjUiv332WbgO4NZbw0WQlZXh1s0PPwxHDLp0CX9HffqEz+6/v+V5dukSNjQGDQpHCb78Upo4Udp557Ahkv63CgCtVfKhf9xx4dBsQ7feGga6AbJp7lxp+vRwKqEtDjggHImoqZEOOki6/HKpf/+slgigiJV86G+xRbgCP13//tI774SLwoBcW7s2bAzMnh3GZJg9O9yGePfdYeTF1jjhhDCvnXcORwkuvzzcHrl6tdSxI6cQgFJX8qH/6afhP9d0V10VrtQH8sWCBeG6k7feCqMoTp/e9nltuaU0aZK02WbhotSKvBxUG0AulHzoz5wZhpxNt2hRuEULyGfu4eLTTp2kqVPDHQJr1oTrCCZMaP38Dj88XNS6cmUYIrljx/Cz337ZGYcCQPJKOvTHjavW5Mnh0Ge6Av/VgH9bujQEd3l5GJL5618Pe/offpid+Z9xRpi/exiRcscdwykKAPmpraFfFAOaLlhQP/B//OPwHlAsNtoo7KV37BiGKnYPD1pyD+NO3H9/GMdACqMVSuFIgZTZsMUTJ0rXXSf97nfhVtcuXaTDDgsjHD71VG5+JwDxK4o9/R13rNbrr4ex7qdPD+PJA4isXx8eEb3ppmEDubY2PBL6oYeiRz7Pm9f8yIe9e0unnBIGRho8OJw622yz+H4HAJGSPrzfo0e1li+Xjj9euvPOpCsCClvduAVz5khXXtly/623lk48MWwAnHEGdxYAcSjp0JeqJYUx9H/1q4QLAopI3bDHb78tPfCA9Oij4RbC5owcGY4cXHZZGMaYgYiA7CP0JV1/fXiyGoDcq6mRfv/7MMzxVVc13/eAA8Ljk0eMCNclAGgfQl/hdqdjjkm4IKAE1daGDYCvfU26777w05SuXaUbb2SkTKA9CH1JTz4ZhjQFkB/efDNcLHj22Y1/3q+ftO224e4DHm8MZK6kb9mrs8kmSVcAIN2gQdEjkd2l996r//kHH0hPPy317BkuANxyy+hWRADZR+gDiE3//iHQ586Vbr45HOpP99FH0lZbhbEFdt453CbYr5/0+ONhpEI2BoD2KarD+ytWMMwoUGjcpRdflC66SJoxI7PvHHZYeGTxqFHS6NG5rQ/IR5zTF3sBQDFYtEiaOTMcuZs8ecMHaTXla18Ltwuee24YOwAoZoS+CH2gmLmH23Kffz6cFrjllub7b755GL74gAOkPfcMpw722ks6+OB46gVyidAXoQ+UmlWrwnM2fvc76aabMv/eDjtIf/iDNGxYZs8mAPINoS9CH4D0ySdh4KAZM8KdATU14bkD8+c3/Z2775b69AkPG+LWQRQCQl+EPoDmvfeeNHBgy/0eeUQ69NDc1wO0FffpA0ALBgyIxgxYtSoc5m/MYYeFcQNqauKtD8g1Qh9ASerYMTxOuG4jYPVq6aij6vfp0SPcCcAzPVAsCH0AkNShg/TXv0rr10v77BO9//770g03hD1/M6lvX2n//cOTBIFCUzSh36FD0hUAKAZm0uzZIfxffHHDzz/9VHrmmTDEsFm4CwAoFEVzIV/PntLSpUlXA6AYffppeCjQsmXh9sAPPtiwz/LlUvfu8deG0lTyF/Ix/C6AXNl0U+n008PDg95/PxwF+Otf6/fp0SPs+V96afgcyEeEPgC0klm46M99w8d5X3yxVF7e+KkBIGlFE/qdOyddAYBS9OSTIfwbjgi4xx7RxX8jRnD6EfmB0AeALPjhD0P4P/zwhp898kh4KmDdRsC3viUde2y4TRCIU9GE/lVXJV0BAISBfdyld9+Vbrut8T6PPy5NmyZ16hSmQFyK5ur9Av81ABSx9evDI4N/8xtpyRJpypQN+zzxhPTNb8ZeGgpUyY+9X+C/BoAS89JL0m671X/v0EPD6QGzZGpC4Sj5W/YAoJDsums4DXDiidF7jz4aHvVrJk2enFxtKF6EPgAk6LbbwpX9w4bVf//73w/hv2xZMnWhOBH6AJCwnj2lZ58N5/4bPtxno42kc89Npi4UH0IfAPKEmXT99dKaNfUv6vvVr8Jnt9zCaH9oHy7kA4A8tWSJtMkm9d/beOPwPhf7lTYu5AOAItO7d7jY75Zbove+/DK62G/5crHDg1Yh9AEgz51ySuOj9/XsGW0ADBsmjRzJRgCaR+gDQAHo0CEE+tq10kknbfj5c89JDz4YNgL+8Y/460Nh4Jw+ABSg1avDrX6PPCItWBCe7tfQmjVSRUXspSEGnNMHgBLSsaNUWRn2+idMCEcBrruufp8OHaS33kqmPuQnQh8AisRZZ4XwP+SQ6L1Bg6SJE5OrCfmF0AeAIjNjhnTRRdHrH/0oXOx3113SihXJ1YXkEfoAUIQuvTRc0NenT/TemDFSt27SQQdJNTXJ1YbkEPoAUKQGDZIWLw4X+w0cGL3/9NNSjx7Sccdxi1+pIfQBoMgdeqj07rthMJ8ddojev/tu6ZhjkqsL8SP0AaBEdO8uzZsnzZ8fRvuTpHvuka68Mtm6EJ/YQ9/MJpvZc2Z2YROf9zKzh82s2sz+EHd9AFDsBgyQPv88bARI0gUXhAv9amuTrQu5F2vom9mRksrdfW9JA81su0a6nSjpz6lBB3qYWasHHwAAtGzhwvqv6zYCULzi3tMfLmlqqj1D0r6N9Plc0o5mtrGkfpI+iKc0ACgtdQ/0KUtLArNwFADFKe7Q7ybpo1R7iaS+jfSZLWlrST+R9I9Uv3rMbGzq8H91rgoFgFKxbl391336SN/5Dlf2F6O4Q79GUpdUu3sTy79Y0g/d/VJJb0o6pWEHd5/k7lVtGXcYALAhd+nkk6PXDz0UjgAQ/MUl7tCfo+iQ/hBJCxrp00vSTmZWLmlPSfzJAUAMpkwJT/Hr3Dl676CDEisHORB36E+XdKKZ/VbSsZLmmdnlDfpcJWmSpKWSeku6K94SAaB0lZdLK1dGr2fOlE47LbFykGWxP1rXzHpJOkTSLHdf1P758WhdAMi2mpowal+dKVPqH/5Hsgrm0bru/oW7T81G4AMAcqN79/r37X/ve9L3vy+tX59YScgCRuQDADSqa1fpnXei15Mnh8P/BH/hIvQBAE3aZhtpUYPjsuXlXNVfqAh9AECz+vYNIX/88dF73M5XmAh9AEBG7rwz7PnXKSuTPvkkuXrQeoQ+ACBj77xT//G8W2wRbuljr78wEPoAgFaZN0+6MO05qbfcEvb6TzwxuZqQmdjv08827tMHgGQ0vJdfknr1kpZs8MQUZFvB3KcPACgO3buHw/qvvBK998UX0gMPJFcTmkfoAwDaZciQ+k/q++53628IIH8Q+gCAdisrk2bPjl4PHSqtXp1cPWhcUYT+eeclXQEAYJ99pPvvj1536lT/4T1IXsGH/pAh0pVXJl0FAEAKh/Yvuyx6vcsuydWCDRV86FdUJF0BACDdhRdKgwaF9ttvSx98kGw9iBR86AMA8s/cuVF7q62SqwP1EfoAgKzr0EGaODF6/Z3vJFcLIoQ+ACAnxo0LV/VL0kMPSddem2w9IPQBADlUWxu1f/pT6R//SK4WEPoAgBzq3Dk8pKfODjtI8+cnV0+pI/QBADm1zTbS9On1XyMZhD4AIOdGjZJuvDF63blzcrWUMkIfABCLM86I2qtWSZMmJVdLqSL0AQCxWbcuGlTtBz+QLr002XpKDaEPAIhNWVl4/G6diy+WzOq/h9wh9AEAsereXVq4sP57vXsnU0upIfQBALHbaivJXbrooui9efOSq6dUEPoAgMRccknU3nHHMIAPcofQBwAkpqxMevTR6PW113JxXy4R+gCARH3729LSpdHriy+uP4ofsofQBwAkrmfP+sG/3XbJ1VLMCH0AQF7o2VP65S+j1+ltZAehDwDIGxdeKPXtG9qXXCK9916i5RQdQh8AkDfKyuo/hW/gwHBrH7KD0AcA5JWuXes/la+MpMoaViUAIO+MGiUdd1z0urY2uVqKCaEPAMhLd94Ztbt3T66OYkLoAwDykpn0619Hr++5J7laigWhDwDIW2efHbWPPlqqrk6ulmJA6AMA8tqLL0bt3XdPro5iQOgDAPJaVVX9gXq++CK5WgodoQ8AyHvpj+AdPTq5OgodoQ8AyHtmUr9+of3449KaNcnWU6gIfQBAQXj55ah9++3J1VHICH0AQEHYZBNp2LDQPu20ZGspVIQ+AKBgXHFF1D788OTqKFSEPgCgYAwfHrVnz06sjIJF6AMACsqTT4bp559LCxYkWkrBIfQBAAXlgAOi9oABPHq3NQh9AEBBKS+X7rgjej1hQnK1FBpCHwBQcE44IYS/JF1+ufTaa8nWUygIfQBAQfrnP6P2zjtL69cnV0uhIPQBAAVpwADppZei13V7/mgaoQ8AKFhDh0pHHBG9nj8/uVoKAaEPACho994btffaK7k6CgGhDwAoeKeeGqaLF0uzZiVbSz4j9AEABe/GG6N2+n38qI/QBwAUvM6dpWuvTbqK/Bd76JvZZDN7zswubKHfRDMbGVddAIDCdswxUdssuTryWayhb2ZHSip3970lDTSz7Zrot5+kzdz9gTjrAwAUrq99Taqqil4/8khyteSruPf0h0uammrPkLRvww5m1kHS/0haYGaj4isNAFDoXnghao8YkVwd+Sru0O8m6aNUe4mkvo30OUnSG5KulrSHmZ3ZsIOZjTWzajOrXrx4cc6KBQAUFjPpzjuj18cem1wt+Sju0K+R1CXV7t7E8odKmuTuiyTdIenAhh3cfZK7V7l7VWVlZc6KBQAUnuOPj9rTpknLlydXS76JO/TnKDqkP0TSgkb6vCNpYKpdJWlh7ssCABSTmpqofcUVydWRb+IO/emSTjSz30o6VtI8M7u8QZ/Jkg40s1mSzpB0Tcw1AgAKXLdu0uGHh/avfpVsLfmkIs6FufsyMxsu6RBJV6cO4b/aoM9yScc08nUAADI2frw0fXpor10rVcSaePkp9vv03f0Ld5+aCnwAAHJin32i9umnJ1dHPmFEPgBAUaqokHr3Du0pU6Q1axItJy8Q+gCAojVzZtS+6KLEysgbhD4AoGjttJPUp09ov/ZasrXkA0IfAFDUzj03TB9+ONk68gGhDwAoav/xH1F7/frk6sgHhD4AoKhtv33UnjYtuTryAaEPAChqZlLnzqF9zjnJ1pI0Qh8AUPR++cswXbhQWrky2VqSROgDAIre+PFR+7nnkqsjaYQ+AKDodewYbt+TpOuvT7aWJBH6AICSUBf6998v1dYmW0tSCH0AQEm47rqoPWJEcnUkidAHAJSEykppv/1Ce9Ys6eabk60nCVkPfTPrku15AgCQDVOnRu1x46TVq5OrJQkthr6ZdTSz/VLtcjMb2cJXLjOzS7NSHQAAWbTZZtKzz0av77svuVqSkMmefm9JT6TaFZL+0kL/zSVt3J6iAADIlWHDolH6jjsu2Vrilknor0r9yN1XSVqb/qGZ/dnMNkp7a3NJc7NWIQAAWZY+Mt/ChcnVEbdMQn+9pHVm9j9mtlxSdzP7wsyWm9khko6X9LqZ7Z7qP0RSCQ99AADIdyefHLWvuiq5OuLWmgv5fidplKRaSYdLeiX1/aWSLpL0qJn9XNJKd5+X7UIBAMiWsjJp9OjQ/sMfkq0lTpmE/gGS3N1fd/enJK11979J+iz1ubv7FEknS7pK0p05qRQAgCw6//yoXSqH+JsNfTO7V9L0DOe1Z2raqV0VAQAQg513jtqzZiVXR5xa2tO/QdJwSTKzvc3sFEkdzewkSf1SfSrM7I+SjpH0TUlHm5nlqF4AALKmbmjeU09Nto64NBv6qcP5r0oyhfAfr7An/zNJXSV9Jal76vM9Uof9P5K0f+5KBgAgO446KkzXrpXWrEm2ljhkeiGfu/tVkoZKWuHuQ9x9B4Wr9Fe4+2nuvizVd6akvbNfKgAA2XXBBVF73Ljk6ohLa4fh7SwpfZhdk3R3gz6vS9qtPUUBABCHioro3P6qVcnWEodMQ7+Tmf1Y0qmSxpvZ981sjKRhkq42s+5pfd+VNC3LdQIAkBPjx4fpHXckW0ccKjLos07Sm5JOkORp3+uW+tlMUgcze1vSDEm3uPvUxmYEAEC+2S3t2PSYMdKdRXzjubl7y71amonZVpIOknScpEMk7e/uzzb/reyoqqry6urqOBYFAChS6fec1dZKXbsmV0smzGyOu1e19nutfrSumVWaWZ/099z9fXef4u6HShoSV+ADAJANCxZE7W7dEisj5zJ5tG4XM/uZBZ0lnS7ppKb6u/vr2SwQAIBc23rr6PY9SXrzzeRqyaVMzulL0n8pnK//L4Wr81eb2RxJfSWtbtB3jaS73P2SbBUJAECu3X13uJpfkgYPlr76SupUZGPMtrin7+4rFYL8q9TP2tTrXgoX93WRdEra9HVJZ5tZeY5qBgAg68rLpYceil5vumlyteRKS2PvH2ZmByqMwlelsGe/bd3nqRH4VqamX6Wmv5H0XYVH8gIAUDBGjJBOPz20ly1rvm8hamlP/3ZJt0mqlHS1pIMljWnuC+7+d3d/yrNxWwAAADG7/PKovWRJcnXkQktj7/dx936SPlQYe/82SZc11T27pQEAEL/0w/qXNZV4BSqTq/fLFS74K5PUUWHo3bLwkU2Q1Ct9WveTy6IBAMilUaPC9Lrrkq0j2zK5T79z6qdW0gupdkeFMff7KpwC6KVwFKBP6r2tclEsAABxSA/7aUU0sHyLt+y5e62ZnSFptbtPNrNjJM139zlmdpqkbdz9/JxXCgBATPr3j9rHHisVy1VqmY7Id7ykd8zsOEl/kfSGmU2RdK6kJ3NUGwAAiXnppaj9wAPJ1ZFNze7pm9lohXvz/6Rw//3g1EfflvSlwkV93czsu2lfK1e4xW+au6/LesUAAMRg6FBp442lL7+UJkyQRo5MuqL2a+nw/iWSVilcme8KF/GZpHtSn78rqSb1Xvo8O0l6MPUZAAAF6YgjpFtvlV55JelKsqOlW/YGu/sukvaX9JykcxTC/2hJD0vaWNJdkqrcfWjqZyd3/7q7E/gAgIKWfsveuiI4dp3pOf2pClflv6GwV/+Yu4+UNELSaEnPmqU/mBAAgMK3xRZR+/77k6sjWzJ94M733P1fkmRmA9x9hSS5e7WZ7SVpD0bgAwAUGzOpslJavDg8ha/Qky6jPf26wE+1Fzb4bK27/z3bhQEAkA/SD/GvWJFcHdmQ6eF9AABK0tixUbtbt+TqyAZCHwCAZphJN98cvX7nneRqaS9CHwCAFvzgB1F7u+2Sq6O9CH0AADKQfm5/+vTk6mgPQh8AgAxceGHUvuSSxMpoF0IfAIAMnZ96vNyrryZbR1sR+gAAZOiss6L2e+8lV0dbEfoAAGRo002j9rRpydXRVoQ+AACtMG5cmJ5zTrJ1tAWhDwBAK6Tv7RcaQh8AgFY47bSovXJlcnW0Reyhb2aTzew5M7uwhX59zezluOoCACAT/fpF7cGDk6ujLWINfTM7UlK5u+8taaCZNTeu0TWSusRTGQAAmdt33zBduLD5fvkm7j394ZKmptozJO3bWCczO0hSraRF8ZQFAEDmHn44atfWJldHa8Ud+t0kfZRqL5HUt2EHM+so6SJJ5zY1EzMba2bVZla9ePHinBQKAEBTevSI2i+9lFwdrRV36NcoOmTfvYnlnytport/2dRM3H2Su1e5e1VlZWUOygQAoHlDh4bp/vtLa9cmW0um4g79OYoO6Q+RtKCRPgdL+pGZzZS0i5n9MZ7SAADI3HnnRe2LLkqujtYwd49vYWY9JT0j6UlJh0k6TtIx7t7olfxmNtPdhzc3z6qqKq+urs52qQAAtGj77aW33w7t9esls3iWa2Zz3L2qtd+LdU/f3ZcpXMz3vKQD3f3VpgI/1X94TKUBANBqU6dG7bvuSq6OTMV+n767f+HuU92dK/MBAAVtyBCpZ8/Q/vnPk60lE4zIBwBAO9SNwf/xx9JbbyVbS0sIfQAA2uEXv4jaf/1rcnVkgtAHAKAdKiqiK/n/93+TraUlhD4AAO3Uu3eYvvBCsnW0hNAHAKCd9toraq9bl1wdLSH0AQBop113jdqvv55cHS0h9AEAaKeuXaOBeebNS7aW5hD6AABkwaGHhunf/pZsHc0h9AEAyIK6i/kmTUq2juYQ+gAAZMHhh0ftVauSq6M5hD4AAFlw1FFRe8KE5OpoDqEPAEAWmEm77RbaN9+cbC1NIfQBAMiSCy4I02XLkq2jKYQ+AABZMnJk1J4/P7k6mkLoAwCQJRUVUfv665OroymEPgAAWfTNb4ZpbW2ydTSG0AcAIItGjAjTe+5Jto7GEPoAAGTRttuG6Rdf5N8FfYQ+AABZVLenL+Xf6HyEPgAAWVRRIR1zTGj//OfJ1tIQoQ8AQJb95CdRe+XK5OpoiNAHACDL9t03ap99dnJ1NEToAwCQA506henttydbRzpCHwCAHKgbf3/5cmnt2mRrqUPoAwCQAyecELWnTEmsjHoIfQAAcqBDB2mPPUL7xhuTraUOoQ8AQI4ce2yYvvJKsnXUIfQBAMiRUaOi9po1ydVRh9AHACBH6obklfLjqXuEPgAAObT55mH63/+dbB0SoQ8AQE5NmBCmn32WbB0SoQ8AQE4dckjSFUQIfQAAcqh//6i9dGliZUgi9AEAyKnycqlfv9C+5ppkayH0AQDIsWXLwjTpkfkIfQAAcuyKK8L0ww8l9+TqIPQBAMixMWOi9oMPJlcHoQ8AQI716hWNwz9+fHJ1EPoAAMTg+OPDdP785A7xE/oAAMRg7Nio/fLLydRA6AMAEIOuXaMheZN61C6hDwBATAYPDtPXXktm+YQ+AAAxGTkyTF98MZnlE/oAAMSkqipqL1wY//IJfQAAYrLvvlH7scfiXz6hDwBAjOr29n/wg/iXTegDABCjuvP6Bx4Y/7IJfQAAYjRsWJg+/XT8yyb0AQCIUd29+kkg9AEAiNGgQVH70EPjXTahDwBAjMrLpV12Ce3HHpPWr49v2YQ+AAAxu+eeqP3oo/Etl9AHACBmAwdKZakEXrkyvuUS+gAAJGD06DBdtSq+ZRL6AAAkoJe/ClgAAAnVSURBVGPHML311viWSegDAJAA9zB94on4lknoAwCQgLPOitp1GwC5Fnvom9lkM3vOzC5s4vONzOwRM5thZveZWce4awQAINeGDo3ad90VzzJjDX0zO1JSubvvLWmgmW3XSLcTJP3W3b8laZGkmIcuAAAg98ykAQNC+y9/iWeZce/pD5c0NdWeIWnfhh3cfaK7P556WSnp03hKAwAgXkcdFaYPPBDP8uIO/W6SPkq1l0jq21RHM9tbUi93f76Rz8aaWbWZVS9evDg3lQIAkGPp5/X/7/9yv7y4Q79GUpdUu3tTyzez3pJ+L+nUxj5390nuXuXuVZWVlTkpFACAXNtyS6lvavd3r71yv7y4Q3+OokP6QyQtaNghdeHeNEnnufvC+EoDACB+11wT37LiDv3pkk40s99KOlbSPDO7vEGf0yTtKukCM5tpZqNjrhEAgNgcfnjUrq7O7bLM47o5sG6BZr0kHSJplrsvau/8qqqqvDrXawkAgBwyi9rr1kXj8jfd3+a4e1VrlxP7ffru/oW7T81G4AMAUAxmzYraZ56Zu+UwIh8AAAnbbz+pT5/Qnjcvd8sh9AEAyAM33BCmPXrkbhmEPgAAeaBXrzB98MHcLYPQBwAgD/RNG67utddyswxCHwCAPLDzzlH7/PNzswxCHwCAPGAmjRsX2rk6xE/oAwCQJ37606h9//3Znz+hDwBAnth226h9xBHZnz+hDwBAHpk2LUzdw+h82UToAwCQR448Mmpn+yp+Qh8AgDxSVibtvXdoDx2a5Xlnd3YAAKC9fvzj3MyX0AcAIM+MGhW116zJ3nwJfQAA8ky3blE7m0+PJ/QBAMhDnTqF6RtvZG+ehD4AAHlom23C9JlnsjdPQh8AgDw0cmSYZnNkPkIfAIA8dPDBYfrll2Ggnmwg9AEAyEO77Ra1TzklO/Mk9AEAyEO9ekXtOXOyM09CHwCAPPX002H6+uvS+vXtnx+hDwBAnho+PGq/8kr750foAwBQAG6/vf3zIPQBAMhjBxwQptdd1/55EfoAAOSxMWOi9hdftG9ehD4AAHnstNOidt++7ZsXoQ8AQB4rL5fGjQvtNWvaN1APoQ8AQJ679tqovcsubZ8PoQ8AQJ7r1Ek6/vjQnju37fMh9AEAKAB33CH179++eRD6AAAUgLIyaf58qbq6HfPIXjkAACCXzOo/iKe1CH0AAEoEoQ8AQIkg9AEAKBGEPgAAJYLQBwCgRBD6AACUCEIfAIASQegDAFAiCH0AAEoEoQ8AQIkg9AEAKBGEPgAAJYLQBwCgRBD6AACUCEIfAIASQegDAFAiCH0AAEoEoQ8AQIkg9AEAKBGEPgAAJYLQBwCgRBD6AACUCEIfAIASQegDAFAiYg99M5tsZs+Z2YXt6QMAAFon1tA3syMllbv73pIGmtl2bekDAABaL+49/eGSpqbaMyTt28Y+AACglSpiXl43SR+l2ksk7dqWPmY2VtLY1MtVZvZ6luvEhvpI+izpIooc6zj3WMe5xzqOx/Zt+VLcoV8jqUuq3V2NH2losY+7T5I0SZLMrNrdq7JfKtKxnnOPdZx7rOPcYx3Hw8yq2/K9uA/vz1F0uH6IpAVt7AMAAFop7j396ZKeMbMtJB0m6Tgzu9zdL2ymz14x1wgAQFGKdU/f3ZcpXKj3vKQD3f3VBoHfWJ+lLcx2Ug5KxYZYz7nHOs491nHusY7j0ab1bO6e7UIAAEAeYkQ+AABKRMGEPiP55V5L68/MNjKzR8xshpndZ2Yd466xGGT6d2pmfc3s5bjqKiatWMcTzWxkXHUVkwz+v+hlZg+bWbWZ/SHu+opF6v+BZ5r5vIOZPWBmz5rZqS3NryBCn5H8ci/D9XeCpN+6+7ckLZJ0aJw1FoNW/p1eo+j2VWQo03VsZvtJ2szdH4i1wCKQ4To+UdKfU7fv9TAzbuNrJTPrJelPCuPXNOVMSXPcfR9JR5tZj+bmWRChL0byi8NwtbD+3H2iuz+eelkp6dN4Sisqw5XB36mZHSSpVmHjCq0zXC2sYzPrIOl/JC0ws1HxlVY0hqvlv+PPJe1oZhtL6ifpg3hKKyrrJI2WtKyZPsMV/VvMktTsxlWhhH7DUfr6trEPmpbx+jOzvSX1cvfn4yisyLS4nlOnTS6SdG6MdRWTTP6WT5L0hqSrJe1hZmfGVFuxyGQdz5a0taSfSPpHqh9awd2XZXAHW6uyr1BCPysj+aFZGa0/M+st6feSWjx3hEZlsp7PlTTR3b+Mrariksk6HippkrsvknSHpANjqq1YZLKOL5b0Q3e/VNKbkk6JqbZS06rsK5RgZCS/3Gtx/aX2QKdJOs/dF8ZXWlHJ5O/0YEk/MrOZknYxsz/GU1rRyGQdvyNpYKpdJYm/59bJZB33krSTmZVL2lMS94fnRquyryDu0zeznpKekfSkUiP5STomfWCfRvrslcFhEaRkuI7HSbpS0qupt25y97vjrrWQZbKeG/Sf6e7D46uw8GX4t9xD0i0Kh0I7SDra3T9qZHZoRIbreA9Jtyoc4n9O0hHuXpNAuQWv7v+B1LU+O7j7DWmfbS3pYUlPSBqmkH3rmpxXIYS+9O+rGA+RNCt1SK5NfdA01l88WM+5xzrOPdZx/kgNW7+vpMda2tktmNAHAADtUyjn9AEAQDsR+gAAlAhCH4DMrFvqKmsARYzQByCFe33Xmpln8HNL3ZfMbP8Mv5P+0y/B3xMoaRVJFwAgL2wlaZWk1anX70j6raSJDfrNlPRJ2us1qWmvDJfxatp3AMSM0Acgd//3uOhmtrukTSQ90HBUQDPbXNL7aW+tTX2/xdEDU2Ow//s7AOLH4X0ADU2QNNvdX0t/08w6Kzxo6d20t9c06PNZI4fzX2wwf0IfSAihD0DSvy/mu03SQZLOSHu/d2oksF8qDKU6t5nZrJB0oLubu5uk/5K0ModlA2gFDu8DJc7MtpR0rEJAr5f07QZ7+eskPaZwsd8V7t7cI5XXZ/gegAQQ+kAJM7NOCs9DL1N4zOwf3b3enrm7LzWzzdz980xmmeF7ABLA4X2ghLn7KoUHdAySNErSisZus5OUfq5+TDOz7Czp6bTvXZt6D0AeYE8fKHHuvizVXCHpHklnN9N9rqSvmvn869pwz54L94A8QegDqLNeUo27L2iqg5mtVzPn6HmcNZDfOLwPoD3asuPAcL9AQtjTB5DuZDM7uYU+6f9vdJCk1Pn7THVodVUAsoI9fQB1XNIdCkPqNvWzTPUvzKuQtLTuvvzmfiQNSPsOgASYe2s20AEgYmYVkrplci7fzMok9VTYSOA/HiABhD4AACWCw/sAAJQIQh8AgBJB6AMAUCIIfQAASgShDwBAiSD0AQAoEf8Pv0T390FyOrUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 绘制精度与召回的曲线,最趋近于右边的（1,1）的切点为最优权衡点\n",
    "\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_precision_vs_recall(precisions,recalls):\n",
    "    plt.plot(recalls,precisions,\"b-\",linewidth=2)\n",
    "    plt.xlabel(\"召回\",fontsize=16)\n",
    "    plt.ylabel(\"精度\",fontsize=16)\n",
    "    plt.axis([0,1,0,1])\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "plot_precision_vs_recall(precisions,recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 通过选择阀值来实现最佳的精度/召回率平衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如果目标定为90%的精度，阀值大概在4000左右\n",
    "\n",
    "y_train_pred_90 = (y_scores > 4000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9137931034482759"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 通过设置阀值为4000，得到大于4000阀值下的精度值\n",
    "precision_score(y_train_5,y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5475004611695259"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 阀值为4000时，对应的召回率\n",
    "recall_score(y_train_5,y_train_pred_90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 总结：\n",
    "   * 获得了一个90%精度的分类器，但如果召回太低，精度再高，也不怎么有用\n",
    "   * 如果工作中，需要99%的精度，你应该回应，召回率是多少？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC 曲线\n",
    "    本质是 真正类率tpr与假正类率fpr （错误的分为正类的负类实例比例）\n",
    "    与召回、精度曲线非常相似"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr,tpr,thresholds = roc_curve(y_train_5,y_scores)     # 实例的标签值,实例的决策值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXxU1eH+8c+Zyb6QBAhLEBEUURGCIYKICoIsLm21KossgqKoxbbauoMiotVqqV/3omxqAbG/utQFEVDcgYAGF1QURBbZIfs6c35/zAQRAwmQzJ3ced6vF2UyuZl5SIUn59xzzzXWWkRERMT9PE4HEBERkdBQ6YuIiEQIlb6IiEiEUOmLiIhECJW+iIhIhFDpi4iIRAiVvoiISIQIeekbY5obY94/yOejjTH/M8Z8aIy5IpTZRERE3CykpW+MSQNmAYkHOex6YIW1tidwiTEmOSThREREXC7UI30fMBjIP8gxvYF5wcfvAdn1nElERCQiRIXyzay1+QDGmIMdlghsCj7eBTTf/wBjzNXA1QCJiYldTzjhhLoNKiIiB1W1g7vd/3/32dnd57f8YqN3u++n9/1KqPRZzD7PWgvlPj8eYyip8OH1GEorfHiNoerAqq8tKqskNspb9ZV785VU+IjyGIwxVPj8dfLnrk+xUR68HoO14LeW2Cgv1lpio734KsrZtXUjFaUlJDRKozh/9w5rbfqhvkdIS7+WCoF4IA9ICn78C9baqcBUgOzsbJuTkxPSgCIi9aW80s/u4nJ2FJZhbaA4fdayp7icLXllFJVV4vEY/H5Lhd/Pl5vzaZUaj99v8QfLourX6p8KaJIYg8cYfNbi91s+3bCHVqnxeEygNP3W4vcHHtvg1327tZBGcVHERHl+fk1/oIjLfH7KK0NXoH4gNvg46QDHJBzg+dQDPO8x4PUYPMZQVumnWXIs0V4PZp/njYGNu0tolRpP48QYPAYMgR84DGCCHxsTeOwJDmYrfZbdxeWc3CqF8ko/cdEeGifGcmLLZGKjvPitpWlSLAkxXhonxtA0KZZorznoYLioqIh27doRHxPNczNnMHjwYIwx62v+7v1aOJb+CuAM4D9AJvCJs3FEJJz4/ZYdhWXsKamgwudne0EZ8PMo0wZHiX4bKLFAmQU+6bdQUu5jR2EZCbFRwaK0+PyBgmzeKHbv8TY4lKwqQ2t/LsmqUe7OonL2FJeTlhATKGe/ZdkPu2icGMN32wrJSIn7xevZva8XyLP/axaVVVLpr/+boO0qKq/xmPzSyhqPiYnyYAgUnscEZnGrBuIej2FPcQVRHkPrxgl7nzdVx+4tzMDH5ZV+dhYFytIbLF+AHYXlZB6Vwo6icjq1SmF3cTnHN0sOlLMn8LVeYyj3+WneKA6PMXg9gdf1GoPXY2icGEOU15AcG01ctKem2eawUVFRQXR0NImJiTz66KOceuqptG3b9ohe09HSN8b0AU6y1j62z9OzgDeMMWcCJwFLHQknIofN57dsLyijqLySSl+gpHcVlZNfWoF3n1HnN1sLSIiJYvVP+TRLjts70vQFR5cr1++mTZMEthWUsWFXMc2S4/gpr4QQ9OIRqSrVzXmlR/Q67dITSYyJwuMxeA38uKuEY5okcEzTRFLio4nyGCxQUFpJu6aJe0ecXs/PJbynuJx26Ul7n4/yGCr9lqZJMcGy/uVoter3KI8hOS4aj+Hn4zw/F3xCTDiOGd3j888/Z8iQIUycOJFLL72UQYMG1cnrOvL/mrW2d/D3xcDi/T633hjTj8Bo/05rrS/0CUUik7WWCp9lT0k5m/eUsml3Cet3FZEYE0WFz8/G3SVYa9lWUEZxuY+EGC9b8kvZll9Gk6QYVm3Mq/NMP+1TnJv2lAAQ4/VQ7vNzYstGRHkMa7cX0vWYxnunXT3G7H0MVUX28+gyr6SC+GgvGcFpbo/HYDDsLCrj+OaBC4Z+MX3LL6dwzT6vX1zuIyU+em8Je4Ij1BaN4gLTwh7z6+ngQCwM+4yQg8d4g2Urkclay1NPPcWNN95IamoqjRs3rtPXD8sf1ay1m/l5Bb+IVKOwrJIP1mznh53FbNhVTKP4aLYXlLFmawGb80rZVVROi0ZxQNW52l9PMVdNf/v8lrySiiPKU1XI+2vfLDDK3FpQSkZKPG2aJJCaEL135LmzsJwOLQLTtS0axeHx/Dyy9AQXYLVKC5RzelIc8TFe0pNjq30vkYZs165dXHXVVfz3v/9l4MCBzJo1i2bNmtXpe4Rl6YtECp/fUl7pZ1tBKV9uzmfznhKKynz4/H581rJuRxFRHg8/5ZVQXhkYaRtjiPaaX4yAD+RARXwwUR5DtNdDo/goEmOiiIv2kpYYTYfmjYjyGnYVldO2aWCrjRaN4kiKi8IGFyfFx3hJio0iPTlW078ih2jhwoW8+uqrPPTQQ9xwww14PHV/Vb3+VoocoT3F5RSWVeLzW3YXV7B+ZxHfby8iNspDpc/y1U95NE6Mxe+3lFX6eOeb7cREefYuQDtSiTFeeh7XlCZJsRzTJIFKvyUlPpp2TRODI2RT/RTzr6avDWkJ0Q1mkZOIG/h8PnJzc8nKymLQoEFkZ2fTrl27ens/lb5EtJ/yStiwq4TNe0pYt6OID7/bQVpizN6pb39w+nvvSnALy37YRaXPT5QncF75SEV5DElxUaTGR5McF83RTRI4Lj2JaG+grPNKKji2aeBipYzUeOKiPTRvFEdCjJcmSZrmFmmoNm3axPDhw/nkk0/45ptvOProo+u18EGlLy5QXF7JzsJyyir9fL0lny15pewuLic2ykulP7BKfP2uYr7clEertHgqfH4+WbvriN9338KPifLQolEcUR7Dxt0lHNU4npYpcWQdnUaUx8OuojJOaNkIr8eQFBvFCS2SSUuIoVF89N5Lk0Qkcrz22muMGjWKkpISnnzySVq3bh2S91XpS1jaVVTO+p1FfPrjHnx+y/fbC/lhZxHpyXEsX7eLVmnxbMkrPeRz1mt3FFX7/AktkkmOi6JZoziy26QFNy/5+fIlzD7XImOI8hoyUuJpkhRDbJSHKK9uWCkiNbPWcuONN/Lwww/TpUsX5s6dS4cOHUL2/ip9CTm/37JpTwk/5ZXy/fZCvt1awHfbCimv9LN0Xe1G4Fvyf72IrV3TRMp9frbll3Fm+6a0TI2jSWIsXk9glXhxeSUtGsVxbLOkwEK1uGjaNk0kJkqFLSKhYYzB6/Xyxz/+kQceeIC4uLiQvr9KX+pMhS+wfeh32wp5fdVPbM0vJdrr4eO1OzmmSSLfbyukoKzmXb6qHNMkgfJKPykJMWS3SSPKa2iSGEO79CSshYzUOKK9Ho5pmkhSrP5TFpHwZK3l2WefpX379px++uk8+OCDji2Y1b+UckgqfX5+yitlXs4GNu8p5aPvd5BXEthq82Dbdn5WvOdXzyXEePF6DG2CC9e6tE7l5FYptEyNJyMlTqvIRaTBKygo4Nprr+Xf//43I0aM4PTTT3f03zaVvlTLWsvG3SUs/2EX324t5LmPf6CovHabI7ZrmkiTpBgq/ZZLuh5Fs+Q4PAaaNwqMzFukxJESrx3HRMTdcnJyGDJkCOvWreOee+7htttuczqSSj/Sbckr5d1vtrF2RxFb8kpJiPEyd/mGGr8uxuvBby1/7NueoxsncFyzJFqlxpOWGBOC1CIi4W3ZsmWcccYZtGjRgiVLlnDGGWc4HQlQ6UcMawMbx7z86Sb+38qNfLk5H4+hxhuXtGmSQHpSLL/rkkHzRnH0PK4piTp/LiJSLb/fj8fjoWvXrowfP55x48bV+f75R0L/ervM5xvz+GTtTorLffxv1WY27S6hwuev9nadVU8dlRZPQoyXHu2a0CIlnvTkWFLio+ndIZ1oXYomIlIrCxcu5MYbb2T+/PlkZGRw5513Oh3pV1T6LvDuN9t4NXcz/125qVbHt2gUx++zWvHbLhm0D96XWkREDk9FRQV33nknDzzwACeccAL5+flkZGQ4HataKv0G7OGF3/LwwjW/et7rMXRtk0aPdk3YUVjGsO5taJUaT6P4KK2IFxGpQ+vWrWPo0KEsXbqUq666iocffpiEhASnYx2QSr8B+fTH3YyYtoxjmibwxab8X33+0q5HcU3vYzk2PcmBdCIikWfSpEl8/fXXvPDCCwwaNMjpODUy1tawkivMZWdn25ycHKdj1DlrLWu2FfKPBd/w1pdbD3hct2MaM2VwJkelhe9PliIiblJcXMyuXbs46qij2LNnD7t376Zt27YhzWCMWWGtzT7Ur9NIP8xYaxk5fRnvr9lR7edjojxc2CWDcWe3p3lKLLFR3hAnFBGJXJ9//jmDBw8mMTGRpUuXkpqaSmpqqtOxak2lH0aqO0ffNCmW45ol8tf+HTjl6DQtuhMRcYC1lqeeeoobbriBtLQ0HnnkETyehnd1k0rfQdZalny7nRkf/sCH3+34xWV1l3Y9insv6qSbwYiIOCwvL4/Ro0fz0ksvMXDgQGbNmkWzZs2cjnVYVPoOsNbywPxveGrJ97/63Jntm/L0yGziojVtLyISDmJiYli/fj0PPfQQN9xwQ4Mc4VdR6YdQSbmPJ979jkcXf/erz/0+qxV3/7YjyXHak15ExGk+n4/HHnuMK664guTkZJYuXUpUVMOvzIb/J2ggxj6X86tV+GPOaMut555AlHa9ExEJGxs3bmT48OEsWbKEuLg4xo4d64rCB5V+vSuv9DNqxjI++n7n3ueOb57Ef649nUYa1YuIhJX//e9/jB49mtLSUmbOnMnIkSOdjlSnVPr16NutBfT/53t7P+7dIZ0Zo07VrngiImHo8ccfZ9y4cZxyyinMmTOHDh06OB2pzqn068Gi1Vu5ctYvNwy6eWAHrut9nEOJRESkJhdccAE//vgjkyZNIjY21uk49UI78tWhmR+uY+L/vvrFczFRHmaOOpXTj2vqUCoREamOtZZZs2bx5ptvMmfOnAa1Kl878jnosw17GPHMUgrKKvc+FxvlYc7Vp5F1dJqDyUREpDr5+flce+21zJ49m969e1NYWEijRo2cjlXvVPpHaFtBKRc+/uHej42BhTf20k1vRETC1PLlyxk6dCg//PAD99xzD7fddhteb2TsjaLSPwLPfbKeCS9/sffjmaNPpXeHhrlLk4hIJKioqGDQoEH4/X6WLFlCz549nY4UUir9w/DR9zu47Omlv3huyqBMFb6ISJjavn07aWlpREdH89JLL9GmTRvS0iLv9GvDWbUQBgpKK+gw/s1fFL7HwPs3n83vs45yMJmIiBzI22+/TadOnZg0aRIAXbp0icjCB5V+rb22ajOdJi6grNIPQPNGsbx4TQ/W/u18WjfWvexFRMJNRUUFt956KwMGDKBx48ZceumlTkdynKb3a+Gj73Ywbvanez++53cdGdHjGOcCiYjIQa1bt46hQ4eydOlSrrrqKh5++GESEjRAU+nXYN2OIoZP+3k6f9ntfWnWKM7BRCIiUpPdu3ezbt065s2bpxH+PlT6B7F5TwlnP/QuAIkxXj65va/ugiciEqaKiop4+eWXGTZsGFlZWaxbt06j+/3onP4BFJRW0DtY+AAv/aGnCl9EJEytWrWK7OxsRowYwZdffgmgwq+GSv8Axj63gvLgor3Ff+nF8c2THU4kIiL7s9byxBNP0K1bN/bs2cPbb79Nx44dnY4VtjS9X415yzfsvRXu7DHdaafd9UREwtKoUaN49tlnOffcc5k5cybNmmm/lINR6e/n84153Pz/VgHQ6/h03ShHRCSM9e/fn8zMTP785z83qBvmOEWlv48Nu4r5zWMf7P146siuDqYREZH9+Xw+7r33Xlq2bMlVV13FsGHDnI7UoOjHoqCt+aWc+fd39n78yh96EhsVGTdgEBFpCDZu3Ejfvn256667WL58udNxGiSN9IOu+/fKvY9fvKYHma1THUwjIiL7evXVVxk9ejRlZWXMmjWLkSNHOh2pQVLpA6+v+okV63cD8PhlWZx6TGOHE4mISJXVq1dz4YUX0qVLF+bOncvxxx/vdKQGK+JLv7TCxx0vfw5Av5Oac37nlg4nEhERgLy8PFJSUjjxxBN56aWXGDhwILGxsU7HatAi/pz+1PfWsqe4AoCHLsl0OI2IiFhrmTlzJm3atOGjjz4C4He/+50Kvw5EdOmXlPt48t3vAXjg4k6kJGjHPRERJ+Xn5zNs2DBGjx7NKaecQps2bZyO5CoRXfrPvL+WkgofGSlxDMpu7XQcEZGItnz5ck455RTmzZvH5MmTWbhwIa1atXI6lqtE9Dn9f7z9LQB3nH8SxhiH04iIRLa3336byspKlixZQs+ePZ2O40oRO9J/5bNNex8P6NjcwSQiIpFr69atfPzxxwDccsst5ObmqvDrUcSO9F/6NFD6nY9KIcobsT/7iIg4ZsGCBYwcOZKYmBi+++47YmJiSE3VHin1KSLbrrzSz7vfbAfgH5dqxb6ISChVVFRw6623MmDAAJo0acLrr79OTEyM07EiQkSO9P+zYiMAGSlxHNdMd9ATEQmV/Px8+vfvz9KlS7n66qv55z//qfveh1BEjvRfyNkAwPAebbSAT0QkhJKTkzn55JOZN28e//rXv1T4IRZxpf/Zhj3kbthDfLSXy3sc43QcERHXKyoq4g9/+ANr1qzBGMMzzzzDpZde6nSsiBRx0/szPlwHwDknNScxNuL++CIiIbVq1SoGDx7MN998Q+fOnWnfvr3TkSJaRI30N+4u5pXPNgPwG+2xLyJSb6y1PP7443Tr1o28vDwWLlzI2LFjnY4V8UJe+saYacaYj40x4w/w+TRjzBvGmBxjzL/q8r0fWbRm7+P+HVvU5UuLiMg+/vWvfzFu3Dj69u1Lbm4uffr0cTqSEOLSN8b8HvBaa3sA7Ywx1c3zjAD+ba3NBpKNMdl19f7/XRm4Nv+O806sq5cUEZF9lJWVAXD55Zczbdo0XnvtNdLT0x1OJVVCPdLvDcwLPl4AnFHNMTuBk40xqUBrYENdvPGi1Vup9FsARp6uGziIiNQln8/H3XffTZcuXSgoKCA+Pp4rrrhCV0iFmVCXfiJQtf/tLqC6/W8/ANoAfwRWB4/7BWPM1cHp/5zt27fX6o1nfvQDELg2PzbKe8jBRUSkehs3bqRPnz5MnDiR7Ow6m5yVehDq0i8E4oOPkw7w/ncB11hrJwFfA6P3P8BaO9Vam22tzT7UaaNLdTc9EZE688orr5CZmcmKFSuYNWsWzz33HMnJyU7HkgMIdemv4Ocp/Uzgh2qOSQM6GWO8QHfAHumbWmt5f80OAH6TmXGkLyciIoDf7+fBBx+kTZs2rFy5kpEjRzodSWoQ6gvVXwbeN8ZkAOcCQ4wxk621+67k/xswg8AU/8fAnCN90x93FQMQ7TUcm554pC8nIhLRvv76a5o0aUJ6ejr//e9/SUlJITY21ulYUgshHelba/MJLOb7BDjbWpu7X+FjrV1mre1orU2y1vaz1hYe6fu+9eUWANKTYrWoRETkMFlrmT59Ol27duXGG28EoFmzZir8BiTk1+lba3dba+dZa7eE6j1fWB64AEDX5ouIHJ68vDwuu+wyrrzySrp3784DDzzgdCQ5DBGxI9/324sAuOiUVg4nERFpeL788kuysrJ48cUXmTx5Mm+//TYZGVof1RC5fvP577YV7H18cqsUB5OIiDRMzZo1o3nz5jz33HOcfvrpTseRI+D6kf4bnwfOIpzWrjFej87ni4jUxtatW7n55puprKwkPT2dDz/8UIXvAq4v/YWrtwLQ89imDicREWkYFixYQOfOnXn00UdZuXIlgBZBu4SrS99ay6qNeQAMPlWb8oiIHEx5eTk333wzAwYMID09neXLl9OtWzenY0kdcvU5/Y27S/Y+btYozsEkIiLhb/To0cyePZuxY8cyZcoUEhISnI4kdczVpV91fX6r1PgajhQRiVw+nw+v18tf/vIXfv/733PxxRc7HUnqiatLf/bSHwHocnSqw0lERMJPUVERf/zjH4mJieHJJ58kKyuLrKwsp2NJPXL1Of2q7Xd7Ha97OYuI7Cs3N5fs7GxmzJhB48aNsfaIb3MiDYBrS7+0wkelP/Af8W91kx0RESCwwPmxxx6je/fu5OXlsXDhQu69916tzo8Qri39qlX7AHHRXgeTiIiEj02bNnHbbbfRt29fcnNz6dOnj9ORJIRce07/u22B+/Rk6Xy+iAhffPEFHTt25KijjmLZsmWccMIJGt1HINeO9NftCJR+Qoxrf64REalRZWUlEydOJDMzk+effx6AE088UYUfoVzbiFXn89s00XWmIhKZNmzYwLBhw3j//fcZMWIEF154odORxGGuLf0FXwa23z1LK/dFJAK9/vrrjBw5krKyMp599llGjBjhdCQJA64sfWstW/NLATi+ebLDaUREnNG2bVvmzJlD+/btnY4iYcKV5/TzSir2Tu8fo+l9EYkQq1evZsaMGQCcf/75LF26VIUvv+DK0q9auZ8UG6XFKiLietZapk2bRnZ2NnfccQeFhYF/A71eXa4sv+TK0t9VVA5AtFeFLyLulpeXx9ChQxkzZgynnXYaOTk5JCUlOR1LwpQrz+lXlf45JzZ3OImISP0pKyujW7dufP/999x3333cfPPNGt3LQbmy9H/YGdhzPynOlX88EYlw1lqMMcTGxnLDDTfQuXNnTj/9dKdjSQPgyun9b7bkA+DR+XwRcZktW7Zw7rnn8uabbwJwzTXXqPCl1lxZ+l5P4I+VGh/tcBIRkbqzYMECMjMzWbJkCTt27HA6jjRAriz9tdsDK1e7aN99EXGB8vJybr75ZgYMGEB6ejo5OTnabEcOiytLf2dwIV/7ZtqYR0QavldeeYUHH3yQa665huXLl9OxY0enI0kD5bqVbkVlleSVVBDj9dAsOdbpOCIih+3HH3/k6KOP5pJLLuHDDz/UuXs5Yq4b6X+9pQCAtMRoPB4t5BORhqeoqIgrrriCjh07sm7dOowxKnypE64b6f+4qwgAax0OIiJyGD777DOGDBnCt99+y+23307r1q2djiQu4rqRfn5JJQCdj0pxOImIyKF57LHH6N69O/n5+SxcuJDJkycTFeW6sZk4yHWlv6OwDICOGSp9EWlYPv/8c/r160dubi59+vRxOo64kOt+hPzgu8C1q80aaRGfiIS/JUuW0KhRI0455RQeffRRoqOjdaMwqTeuG+lHe6s25olxOImIyIFVVlZy11130adPH8aPHw9ATEyMCl/qletG+svW7QLgxJa6Rl9EwtOGDRsYNmwY77//PiNHjuSxxx5zOpJECFeVfuAmFIGV+63S4p2OIyLyK1988QVnnXUWFRUVPPfccwwfPtzpSBJBXDW9X1rhx1qI8XqIjdLtJUUk/JxwwgkMHjyYlStXqvAl5FxV+vmlFQCU+/wOJxER+dnq1as599xz2bFjB1FRUTz55JO0b9/e6VgSgVxV+kVlgWv0j9LUvoiEAWst06ZNIzs7mxUrVvD99987HUkinKtKP780UPqpCbqlrog4Ky8vj6FDhzJmzBh69OhBbm4u3bt3dzqWRDhXlX5hsPQbxan0RcRZN910E//5z3+47777WLBgAS1btnQ6koi7Vu9/vSUfgLQEXaMvIqHn9/vJy8sjLS2Ne++9l9GjR9OjRw+nY4ns5arSLyn3AeDz6247IhJaW7ZsYeTIkRQXF/Puu++Snp5Oenq607FEfsFV0/tfbM4DoEMLbcwjIqHz1ltvkZmZuXezHa9XlwxLeHJV6f+4qwSAmChX/bFEJEyVl5dz0003MXDgQJo1a0ZOTg5XX321ttKVsOWqdkyOC5ytaNs00eEkIhIJysrKePnll7nmmmtYtmwZHTt2dDqSyEG56pz+Zz/uAeD45kkOJxERN3vllVfo378/ycnJrFixgkaNGjkdSaRWXDXSr9qJr1G8LtkTkbpXWFjI6NGjufDCC3n88ccBVPjSoLhqpB/tNVT4rK7TF5E699lnnzF48GDWrFnDhAkT+POf/+x0JJFD5prS9/ktFb7AXfZitZBPROrQCy+8wMiRI2natCmLFi3i7LPPdjqSyGFxTTtW7cZnLVo5KyJ1Kisri4suuojc3FwVvjRorin97YWlACTE6PpYETlyS5Ys4Y9//CPWWtq3b8/cuXNp2rSp07FEjohrSj9/n5G+iMjhqqys5K677qJPnz7Mnz+fnTt3Oh1JpM64pvQ37CoGoHu7xg4nEZGGasOGDZx99tlMmjSJESNGsHLlSo3uxVVcs5Bve0EZAF6dzxeRw+Dz+TjnnHPYvHkzzz//PMOGDXM6kkidc03pV03rp+gafRE5BKWlpURHR+P1epk6dSqtWrXiuOOOczqWSL1wzfT+6p8Ct9VtlRbvcBIRaShWr15Nt27dePDBBwHo1auXCl9czTWln5oQA0BeSYXDSUQk3FlreeaZZ+jatStbtmwhMzPT6UgiIRHy0jfGTDPGfGyMGV/DcU8YY35T29fN3RjYd//EltoSU0QOLC8vj6FDh3LVVVdx+umnk5uby7nnnut0LJGQCGnpG2N+D3ittT2AdsaY9gc47kyghbX2f7V97fSkWAAqg/vvi4hU56uvvuLll1/mvvvuY8GCBbRs2dLpSCIhE+qRfm9gXvDxAuCM/Q8wxkQDTwM/GGN+V9sXrprWb9NEt9UVkV/y+/0sXrwYgB49evDDDz9w22234fG45gynSK2E+r/4RGBT8PEuoHk1x4wEvgL+DnQzxly//wHGmKuNMTnGmJzt27cDUFAWKH3dYU9E9vXTTz/Rv39/+vbty4oVKwBo0aKFw6lEnBHq0i8EqpbXJx3g/U8BplprtwDPA7/a6NpaO9Vam22tzU5PTwegvDIwrR8XrZ/cRSRg/vz5ZGZm8tFHH/H000+TlZXldCQRR4W6IVfw85R+JvBDNcd8B7QLPs4G1tfmhb/dWghAtFelLyIwfvx4zj33XFq0aEFOTg5jxozRzbgk4h1xQxpjPMGFd7XxMjDCGDMFGAR8aYyZvN8x04CzjTHvAdcBD9XmhRsnBi7Zi1HpiwjQunVrrrvuOpYuXcpJJ53kdByRsFDjjnzGmBjgL8D9QJy1tilH2DsAACAASURBVCT4fBwwmMDCvLeAhJpey1qbb4zpDfQD/h6cws/d75gC4NJD+2OAP7gln+6yJxK5Zs+ejdfrZfDgwYwdO9bpOCJhpzbDYg9wE3A9cOc+zz8P3A4YoNY74lhrd1tr5wULv87sKQ5EiItW6YtEmsLCQkaNGsWwYcOYNWsWVrfbFKlWbfbeLweKgDeAHGPMx0B7ApffdbXWFhtjfPUXsWZ+/89/wVX6IpHl008/ZciQIaxZs4YJEyZw55136ty9yAHUWPrWWr8xpsJa+50x5gbgR+BTYBnwO2PMvIO/Qv0rrfz5Zw6vR3/ZRSLF2rVrOe2000hPT2fx4sX07t3b6UgiYe1Q77K3xVr7mTHmFOAR4CTg47qPdWiKygKln5aga/RFIkFlZSVRUVG0a9eO//u//+OSSy7Rfe9FaqHWS92NMd2A/2eMGUjgUrq1wFZr7XIC5/UdU1oRKP3dxbrZjojbvfvuuxx//PF8+umnAFxzzTUqfJFaOmjpG2NOM8a8EvzwU+BBApfd7SKwwj4tePldvDFmSvDXw8aYp+o19X7KgtP7x6ZrC14Rt6qsrOTOO++kT58+REdHawtdkcNQ0/R+OwJb50YDLwETgT8RuJbeAvnAsQR+eGgb/BovEFcPWQ+oano/NkqL+ETc6Mcff2TYsGF88MEHjBo1ikcffZSkpCSnY4k0OActfWvtbGC2MWYjgYJ/gEDZ9wVeIXBt/pXAGmvtRfWc9YCKywOlvyW/1KkIIlKPZsyYQW5uLs8//zzDhg1zOo5Ig1Xb+bFya+1lwG4gBSgFLgEaAW0I/CDgmKqNeY5Ki6/hSBFpKEpKSvjqq68AuP3221m1apUKX+QIHepJsaeAE4GdBKb+s621K+o81SGqutlOWkKMw0lEpC589dVXdO/enf79+1NSUkJ0dDTHHHOM07FEGrwaS98EdrmINcY0BuYSOL+fSOCSvWb1G692NueVABATpYU9Ig2ZtZapU6eSnZ3N1q1beeaZZ4iP1wyeSF2pzXX6sQTO3Q8E5lhrvwAwxowEnjXGnA44OsSuusnOhl3FTsYQkSNQUlLC5Zdfzosvvsg555zDc889p/vei9Sx2gyNK4FxBEb5t1Y9aa19E3gY8BP4wcAxm/cEFvBltUlzMoaIHIHY2FjKysq4//77eeutt1T4IvWgNtvwVgL/Dn5YtN/n/hac/u9aD9kOWUFppdMRROQQ+P1+pkyZwqBBgzj66KN5+eWXtW++SD064pPgNmBVXYQ5XMXlgbLPSA3p9gAicgR++ukn+vfvz0033cSsWbMAVPgi9axWpW+MiTXG/NcYExv8uKkxppkxJtEY4zPGJO5z7LPGmJ71Fbg6G3YHzuU3itPe+yINwZtvvklmZiYfffQRTz/9NOPHj3c6kkhEqGkb3qofu/3A74K/A0wH3gIqCOy7XxY8vhEwBMioj7AHkhyrshdpKObOnct5551HixYtyMnJYcyYMRrhi4RITSP9V4wxv7XWVgBYayuMMVcRWMn/F2tteeBpW3UyfSSBDXxerrfE1SgoC9xop3XjhFC+rYgcAhvcROv888/nzjvvZOnSpZx00kkOpxKJLAcsfWOMh8BNduYEL8/DGNMa+Adws7V28X7HxwF/Bu6q+iEhVKoW8CXHHeqdgkUkFJ5//nnOOOMMSkpKSE5O5u6779b19yIOOGDpW2v91tq7CNxNb0Tw6UeApdbah6v5kr8BPwFT6zxlDXYWlgMQpxvuiISVwsJCRo0axYgRI/B4PBQUFDgdSSSi1eaSvTeAN4wxfuAWoBAC5/ttYL7OGGP+AVwInGat9R/41erHxuBCviivzguKhItPP/2UIUOG8N1333HnnXcyYcIEoqI0GyfipIP+DTTGzAeqtrmzwP2AJ7iKf48xplvwc78Belhrt9Zb0oNomRJPfmkBCTEa6YuEA2st1113HUVFRSxevJhevXo5HUlEqHmkv5LgynwCI/kTgRcIbLu7GfgI+D/gKOBOY8yfQn0+H6DcF5hciItW6Ys4aceOHURFRZGamsrs2bNJTk6madOmTscSkaCDrt631t5urb2bwOI9CNxKNyn4/GPW2kcJzAB0AU4Fnq7XtAdQUBr4OaNqD34RCb133nmHzp07M27cOADatm2rwhcJM7W5y97fgIUEyv1MYJgxZty+x1hrvyVwHf+5xpjf1kfQg9kRXMinu+yJhF5lZSUTJkygb9++NGrUiL/+9a9ORxKRA6hpc54bgTHAnwCstWuBYcDfjDHtqg4Lfm4zgXP+d9Vb2gOIDZa9LtkTCa0NGzbQq1cvJk+ezKhRo1ixYgVdunRxOpaIHEBNQ+MvgAuAZRC4dj94ff5rwEPVHD8LONkYc3KdpqxBWWXwnL4u2RMJKY/Hw5YtW5g9ezbTp08nMTGx5i8SEcfUdE5/gbV2KYGFe4bAOX0IjOh/a4zpAIG9+YPH7yKwoc9F9ZZ4P/7gLl8AHo8u2ROpbyUlJTzyyCP4/X5atWrF119/zdChQ52OJSK1UNuT4JbAKn0/gLU2FzgNWA8sITjFHzQHWFSHGQ8ezNZ8jIjUjS+//JJu3brxpz/9iXfffReA6Gjd+0KkoahV6Vtry621N1hr8/d5LsdaW2qtPdtaW7rP8/9nrf2oPsJWmy34e9Ok2FC9pUjEsdYydepUTj31VLZt28b8+fPp06eP07FE5BA1+OXuVTfxiNFufCL15oYbbmDs2LH07NmT3NxcBgwY4HQkETkMNS53N8ZEAS2ttRtqceyxwP3W2kvrIlxt+IND/WhdridSby699FJatmzJTTfdhMejv2siDVVtrnHrDHwA7L1vrTGmBfAGcPq+U/tAEoHb7oZM1UK+9TuLazhSRGrL5/PxwAMPkJ+fz/3330/Pnj3p2bOn07FE5AjV5kf2UmD/rXUrgEygfL/ny6s5tl5VLeTrfFRKKN9WxLU2b95M//79ueOOO1i/fj1+f8jvoSUi9aQ2pe8L/tpXJQRuv7vf8yH/16FqpK9990WO3BtvvEFmZiYff/wxzzzzDLNnz9Z0voiLNPgt7HzBk/oFpZUOJxFp2LZt28Yll1xC+/btmTt3LieeeKLTkUSkjjX40q9as6/V+yKHZ+vWrTRv3pxmzZoxf/58unXrRlxcnNOxRKQe1HbeLsUYs7bqF5ALmH2fCz6/sP6iVq/qOv2WKfGhfmuRBu/555/nuOOOY86cOQCcddZZKnwRF6vtSL8UuLsWx2UANx1+nMPn1UhfpNYKCgoYN24czz77LGeeeSZnnHGG05FEJARqW/pl1tpZNR0U3Is/pKVftXo/Svvui9TKypUrGTJkCN9//z0TJ07kjjvuICqqwZ/pE5FaaPB/021wgt+r0heple+//56SkhLeeecdzjrrLKfjiEgIHXLpG2PGAGfy68v4AEJ+sXzVjnxeo9IXOZDt27fzySef8Jvf/IZLL72U8847T7fBFYlAtSl9wy8X/CUAjQleq7+fpLoIdSh8fosHKK6o7mcQEXnnnXcYNmwYRUVFrF+/ntTUVBW+SISqTenHBX8BYK19BHikugONMScCIbvDHkDVrL7fr3vsiuyrsrKSiRMnct9993H88cfzxhtvkJqa6nQsEXFQjaVvrf2MfUq/BjGAI9fOZaTqkj2RKhUVFfTp04cPPviAK664gkceeUSjexGpm1vrGmM6G2O8wOdA87p4zdrau3pfl+yJ7BUdHc3AgQOZPXs206ZNU+GLCFCL0jfGdDfGHPC4YNl/CqQDXqBl3cWrWdWkvi7Zk0hXUlLCddddx7vvvgvAHXfcwdChQ50NJSJhpTYj/TkcZHrfWusjsNivDBgOLAz+IBAS1lZdsqebgkjk+vLLL+nWrRtPPvkkS5cudTqOiISp2izkKwfKjDETgx9Xdyc9S+ASvj8D/wn+IBASZZV+Evh5D36RSGKt5emnn+bPf/4zycnJzJ8/nwEDBjgdS0TCVG1Kv6rk/wSsAs4APgFOA9bw8/X6nYBjgT51nPGgYryBEf6uovJQvq1IWHj11VcZO3Ys/fr149lnn6VFixZORxKRMHYoc+IW6E9gUP374O9TgEnBxxcCL1hrd9Z1yJpCAbRpkhDKtxVxVEFBAQC/+c1vmDt3LvPnz1fhi0iNDudEuOXnrt33uaeAfxxxokMNEzynr4V8Egl8Ph/33nsvxx57LD/++CMej4fBgwfj0ZoWEamFA07vB1fsP01g972zCKzM3/vpar5ku7U2v27j1azqpw+vV//oibtt3ryZ4cOH88477zB06FBSUkK+67WINHAHO6cfTeBWuUnAGwQ23gk7vuBOfNEa6YuLvf7664waNYri4mKmT5/OqFGjMLrfhIgcogMOj621Zdbac4EfCRR/Xg2vdYIx5tK6DFcblb5A6VdoG15xsblz55KRkcGKFSsYPXq0Cl9EDktt77JnD/D7vvoBo4AXjzDTIanaiU/n9MVt1qxZg9/vp0OHDjz55JNERUURF1fbHbFFRH6ttifCTfDX0uDvC4PP3wHcH3z8NBBjjDm3ThPWoGob3tT46FC+rUi9eu6558jKyuKaa64BICkpSYUvIkfsUEb6k4OPZ+73OUNg1X4p8E/gKuDNA72QMWYacBLwurV28kGOaw7Mt9aeUpuAXo30xQUKCgr4wx/+wHPPPcdZZ53Fs88+63QkEXGR2pR+DBBnra32cjwTOLn4DwKr+58F7jLGRFtrK6o59veA11rbwxgz3RjT3lq75gDv+xC1uGPf3kv2dMMdaeDWrVtH//79Wbt2LRMnTmT8+PF4vSHb0VpEIkBtSv9xft51rzpxBEb7sdbaLcaYPtUVflBvYF7w8QICu/v9qvSNMX2AImBLTeH2XrKn65SlgcvIyODEE09k2rRpnHXWWU7HEREXqrEprbX/tNaWHeTzJUBbYGvw408P8nKJwKbg411UcxteY0wMMAG49UAvYoy52hiTY4zJKSqrBMCr1czSAG3fvp2xY8eSl5dHbGwsr776qgpfROpNnQyPrbXrbdU8+8EV8vOUfdIB3v9W4Alr7Z6DvN9Ua222tTY7PiYw/emv1duLhI/FixeTmZnJzJkz+eSTT5yOIyIRINRz4isITOkDZAI/VHPMOcAfjDHvAl2MMc8c7AWruj5Fq/elgaisrOSOO+7gnHPOoVGjRixbtkx3xhORkKjt6v268jLwvjEmAzgXGGKMmWytHV91gLV279ymMeZda+2Yg71g1fheC/mkobj55pv55z//yRVXXMEjjzxCYmKi05FEJEKEtPSttfnGmN4ENvL5u7V2C5B7kON71+I1AYjSQj4Jc+Xl5cTExPCXv/yF7t27M3jwYKcjiUiECXlTWmt3W2vnBQv/iFX4dMmehLfi4mLGjh3LBRdcgN/vp1WrVip8EXFEgx8eV430PVq9L2Hoiy++oFu3bkydOpWsrCz8fr/TkUQkgoX6nH6dq5rWj41q8D+/iItYa/nXv/7FDTfcQEpKCgsWLKBfv35OxxKRCNfgm9JSdU5fI30JH4WFhdx333306tWL3NxcFb6IhIUGP9KvWr3vUelLGFixYgWdOnUiOTmZDz/8kFatWuHRIlMRCRMN/1+jYOtrpC9O8vl83HvvvXTv3p0HH3wQgNatW6vwRSSsaKQvcoQ2b97M8OHDeeeddxg6dCjXX3+905FERKrlgtLXOX1xzuLFixk8eDDFxcVMnz6dUaNGYXQliYiEqYZf+sGhvlelLw5IT0/nuOOOY8aMGZxwwglOxxEROSjXnHDUjnwSKt9++y1/+9vfAOjUqRMfffSRCl9EGgTXNKVG+hIKzz77LFlZWTz00ENs3rwZQNP5ItJguKL01fdS3woKChgxYgSXX345Xbt2JTc3l4yMDKdjiYgckgZ/Th80ypf6Za2lT58+rFy5kokTJzJ+/Hi8Xq/TsUREDpkrSl/77kt98Pv9GGMwxjBhwgRSU1M566yzav5CEZEw5YrpfY30pa5t27aNCy64gMceewyA3/72typ8EWnwXFH6GulLXVq0aBGZmZksXryYmJgYp+OIiNQZl5S+0wnEDSoqKrj99tvp168faWlpLFu2jLFjxzodS0Skzrii9DW9L3UhJyeH+++/nyuvvJLly5fTuXNnpyOJiNQpVyzkU+nLkVi9ejUnnngiPXr0IDc3l06dOjkdSUSkXrhipL+jsNzpCNIAFRcXM3bsWE4++WSWLl0KoMIXEVdzxUg/IyXO6QjSwHzxxRcMGTKEL7/8kltuuYWsrCynI4mI1DtXlL5uqyuH4plnnuH6668nJSWFBQsW0K9fP6cjiYiEhCum93VOXw5FXl4evXr1Ijc3V4UvIhHFHaWv6/SlBh9++CHz588H4IYbbuCNN96gefPmDqcSEQktV5S+pvflQHw+H5MnT6ZXr15MmDABay0ejwePbsUsIhHIFf/yaaQv1dm0aRPnnHMOEyZMYNCgQSxatEi3wRWRiKaFfOJKmzZtIjMzk5KSEmbMmMHll1+uwheRiOeK0ve6Yr5C6oK1FmMMGRkZjBs3jiFDhnDCCSc4HUtEJCy4oi7XbS9yOoKEgW+//ZazzjqL1atXY4xh4sSJKnwRkX24ovTbpic6HUEcZK1l1qxZZGVl8dVXX7F582anI4mIhCVXlH5CtCvOUshhKCgoYMSIEYwaNYrs7GxWrVpF3759nY4lIhKWXFH6Wp8Vuf75z38yZ84cJk2axKJFi2jVqpXTkUREwpYrhsgetX5E8fv9bNmyhYyMDG655RYGDhxIt27dnI4lIhL2XDHS1z4rkWPbtm1ccMEF9OzZk8LCQmJjY1X4IiK1pJG+NBiLFi1i+PDh7N69mylTppCYqAWcIiKHQmNkCXuVlZXcfvvt9OvXj7S0NJYtW8Z1112nzXZERA6RK0pfI313M8bw0UcfMWbMGJYvX07nzp2djiQi0iC5ZHrf6QRSH/773/9y+umn06JFC+bPn09cXJzTkUREGjSN9CXsFBcXc/XVV3PxxRfz4IMPAqjwRUTqgCtG+jq36x6ff/45Q4YMYfXq1dx6661MmjTJ6UgiIq7hktJ3OoHUhfnz53PRRReRkpLCW2+9Rb9+/ZyOJCLiKi6Z3nc6gdSFbt26MWTIEHJzc1X4IiL1wCWlr9ZvqD744AMuueQSysvLady4MTNmzKB58+ZOxxIRcSVXlP6Pu4qdjiCHyOfzcc8999CrVy8+++wzNm3a5HQkERHXc0Xpt0qNdzqCHIJNmzZxzjnncOeddzJkyBBWrlxJ27ZtnY4lIuJ6rljIlxDjdTqCHIKhQ4eycuVKZs6cyciRI3X1hYhIiLii9HVOP/yVlZXh8/lISEjgqaeewuv10qFDB6djiYhEFFdM72ukGN6++eYbTjvtNK6//noATjrpJBW+iIgDXFH6umQvPFlrmTlzJl27dmXDhg1cdNFFTkcSEYloLil9tX64yc/PZ/jw4YwePZpTTz2V3NxcLrjgAqdjiYhENHeUviv+FO6yc+dO5s+fzz333MPChQtp1aqV05FERCKeKxby6Zx+ePD7/bz88stcdNFFtG3blu+//57U1FSnY4mISJArxsiqfOdt27aN888/n4svvpjXX38dQIUvIhJmXDHS1zl9Zy1cuJARI0awe/dunnjiCc4//3ynI4mISDVcMdLX6n3n/P3vf6d///6kpaWxfPlyrr32Wp1uEREJU64ofZWMc0455RTGjBlDTk4OnTp1cjqOiIgchKb35ZDNmzeP9evXc9NNN9GvXz/dBldEpIFwxUh//c4ipyNEhKKiIq666ioGDx7MK6+8QmVlpdORRETkEIS89I0x04wxHxtjxh/g8ynGmDeNMQuMMS8ZY2Jqes1jmyXVfVD5hVWrVpGdnc20adO47bbbeOedd4iKcsVEkYhIxAhp6Rtjfg94rbU9gHbGmPbVHDYMmGKt7Q9sAQbW+Lp1G1P2s3v3bs444wz27NnD22+/zX333Ud0dLTTsURE5BCFeqjWG5gXfLwAOANYs+8B1ton9vkwHdi2/4sYY64GrgaIaXGcWr+elJSUEB8fT1paGrNmzaJnz540a9bM6VgiInKYQj29nwhsCj7eBTQ/0IHGmB5AmrX2k/0/Z62daq3NttZmAxi1fp374IMP6NChA6+88goAF110kQpfRKSBC3XpFwLxwcdJB3p/Y0xj4FHgitq8qBbv1x2fz8ekSZPo1asXMTEx2jNfRMRFQl36KwhM6QNkAj/sf0Bw4d6LwG3W2vW1eVF1ft3YuHEjffv25a677mLo0KGsXLmS7Oxsp2OJiEgdCXXpvwyMMMZMAQYBXxpjJu93zJVAFnCHMeZdY8zgml5UI/26sXjxYnJycpg1axbPP/88jRo1cjqSiIjUIWOtDe0bGpMG9APes9ZuOdLXi23Z3t478zX+OqDDkYeLQGVlZaxcuZIePXpgreWnn34iIyPD6VgiInIQxpgVVevaDkXIr9O31u621s6ri8KvopH+4fnmm2847bTT6NevH9u3b8cYo8IXEXExV+zIJ4fGWsvMmTPp2rUrGzZsYO7cuaSnpzsdS0RE6pkrSl8D/drz+XyMGDGC0aNHc+qpp5Kbm8sFF1zgdCwREQkBV5S+5vdrz+v10qxZM+655x4WLlyoS/JERCKIKzZPV+UfnN/vZ8qUKZx55pl0796dKVOmOB1JREQc4IqRvgb6B7Z161bOO+88brrpJubMmeN0HBERcZBLRvpq/eosWLCAkSNHkpeXx5NPPsnYsWOdjiQiIg5yR+mr839l0aJFDBgwgJNOOomFCxdy8sknOx1JREQc5o7pfacDhBGfzwdA7969eeihh1i+fLkKX0REALeUvlofgBdeeIGTTjqJLVu24PV6+ctf/kJCQoLTsUREJEy4pPQju/WLiooYM2YMQ4YMoXHjxlRUVDgdSUREwpArSj+SrVq1iuzsbKZPn87tt9/Oe++9R+vWrZ2OJSIiYcgVC/ki2X333UdeXh5vv/02ffv2dTqOiIiEMVeUfqTN7u/atYuioiJat27NE088gc/n0975IiJSI1dM70fSdfrvv/8+Xbp04bLLLsNaS+PGjVX4IiJSK+4o/QjofJ/Px6RJk+jduzexsbE8/PDDEb+AUUREDo07pvedDlDPtm3bxqBBg1iyZAnDhw/niSeeIDk52elYIiLSwLij9F3e+omJiRQXFzNr1ixGjhzpdBwREWmg3DG978KxfmlpKZMnT6aoqIjExEQ++eQTFb6IiBwRd5S+yzr/66+/5rTTTmPChAm89tprAHg8rvi/SkREHKQmCSPWWqZPn07Xrl3ZtGkTr732GoMHD3Y6loiIuIQrSn9rfqnTEerE5MmTufLKK+nevTu5ubmcf/75TkcSEREXccVCvlap8U5HOCLWWowxDBs2jJiYGP7617/i9XqdjiUiIi7jipF+Q71e3e/38+CDDzJo0CCstbRr145bbrlFhS8iIvXCJaXvdIJDt3XrVs477zxuvvlm/H4/paXuOEUhIiLhyxWl39AsWLCAzMxMlixZwlNPPcV//vMf4uMb9ikKEREJf644p9+QBvrFxcWMGjWKJk2asHDhQk4++WSnI4mISIRwRek3hPn9DRs2kJGRQUJCAm+99RbHHnssCQkJTscSEZEIoun9EHjhhRc4+eSTeeCBBwDo1KmTCl9ERELOFaUfruP8oqIixowZw5AhQ+jYsSOXXXaZ05FERCSCuaP0w7D1P//8c7Kzs5k+fTq33347S5Ys4ZhjjnE6loiIRDB3nNMPQyUlJRQXF7Nw4UL69OnjdBwRERGXjPTDZIJ/586dTJ8+HYBu3bqxZs0aFb6IiIQNd5R+GHT+e++9R5cuXbj22mv54YcfAIiJiXE2lIiIyD7cUfoOvndlZSUTJ07k7LPPJj4+no8//ljn7kVEJCzpnP4RsNby29/+ljfffJMRI0bw+OOPk5yc7HQsERGRarmi9J2a3jfGcNlllzF06FBGjBjhTAgREZFackfph3CCv7S0lJtuuomsrCxGjx7N8OHDQ/beIiIiR8IV5/RDZfXq1XTv3p3HHnuM7777zuk4IiIih8QVI/36Huhba5kxYwbXX389CQkJvP7665x33nn1+6YiIiJ1zBUj/fqe3M/JyeHKK6+ke/fu5ObmqvBFRKRBcsdIv55s376d9PR0Tj31VObPn88555yD1+t1OpaIiMhhccdIv46X7/v9fv7+979zzDHHsHz5cgAGDBigwhcRkQbNFSP9uqz8rVu3MnLkSBYsWMDFF1/McccdV4evLiIi4hxXjPTryoIFC8jMzOS9997jqaee4sUXXyQtLc3pWCIiInXCHSP9Ohrqf/zxxzRt2pRFixbRsWPHunlRERGRMOGKkf6RlP7atWv58MMPARg/fjzLly9X4YuIiCu5o/QP86z+nDlz6NKlC2PGjMHn8+H1eomPj6/jdCIiIuHBFaV/qIqKirjiiiu47LLL6NSpE/Pnz9fKfBERcb2IO6e/fft2zjzzTL799lvGjx/PXXfdRVSUK74NIiIiBxVxbde0aVPOPvtsnnzySc4++2yn44iIiIRMREzv79y5k2HDhrF27VqMMSp8ERGJSK4o/YPtyLdkyRIyMzN58cUX9+6uJyIiEoncUfrVPFdZWcnEiRPp06cPCQkJfPLJJwwePDjk2URERMKFK0q/OlOmTOHuu+9m+PDhrFixgqysLKcjiYiIOMoVC/n2nd0vLCwkKSmJP/zhDxx77LFcfPHFzgUTEREJI64Y6RsMpaWljBs3jm7dulFUVERiYqIKX0REZB8hL31jzDRjzMfGmPFHcsy+NqxdQ/fu3Xn88ccZOHCgrrsXERGpRkhL3xjze8Brre0BtDPGtD+cY/blK87nz0MHsnnzZl5//XWmTJlCbGxs/fwBREREGrBQj/R7A/OCjxcAZxzmMXv5S/I4oXNXcnNzOe+8yZab1AAACU1JREFU8+oopoiIiPuEeh48EdgUfLwLqG5JfY3HGGOuBq4OfliWu+yDL1q1alXHUWU/TYEdTodwOX2P65++x/VP3+PQ6HA4XxTq0i8Eqm5jl0T1Mw01HmOtnQpMBTDG5Fhrs+s+quxL3+f6p+9x/dP3uP7pexwaxpicw/m6UE/vr+Dn6fpM4IfDPEZEREQOUahH+i8D7xtjMoBzgSHGmMnW2vEHOea0EGcUERFxpZCO9K21+QQW6n0CnG2tzd2v8Ks7Jq+Gl51aD1Hl1/R9rn/6Htc/fY/rn77HoXFY32djra3rICIiIhKGXLEjn4iIiNSswZR+fezkJ79U0/fPGJNijHnTGLPAGPOSMSYm1BndoLb/nRpjmhtjPg1VLjc5hO/xE8aY34Qql5vU4t+LNGPMG8aYHGPMv0Kdzy2C/w68f5DPRxtj/meM+dAYc0VNr9cgSr8+dvKTX6rl928YMMVa2x/YAgwMZUY3OMT/Th/i58tXpZZq+z02xpwJtLDW/i+kAV2glt/jEcC/g5fvJRtjdBnfITLGpAGzCOxfcyDXAyustT2BS4wxyQd7zQZR+tTDTn7yK72p4ftn/397dx8jV1XGcfz763bpgiJFkEJMoZoaXqvbRltAo1ugKS9RStCQCrHFGCxBoECitEkFTRRfksZqKbEoGIIiilps5MVWWcFoUTRWKzVItKS2+BorlJbWpY9/nDO7t5PZnTv7MnVmf59k0pl7z9w5PZ3Oc+859zwnYk1EbMgvXwf8vTlVays9lPieSjoHeIl0cmWN6aFOG0vqBO4Etkm6uHlVaxs91P8e/ws4Q9JkYCqwvTlVayuvAJcBLwxRpoeBf4vHgSFPrlol6Fdn6ZsyzDI2uNLtJ+ks4OiI2NSMirWZuu2ch01WADc3sV7tpMx3+QPA08DngNmSrm1S3dpFmTb+KXAScB2wNZezBkTECyVmsDUU+1ol6I9KJj8bUqn2k/Ra4EtA3bEjq6lMO98MrImIXU2rVXsp08YzgbUR8VfgXmBuk+rWLsq08S3Akoj4JPAH4Mom1W28aSj2tUpgdCa/sVe3/fIV6LeBZRHxXPOq1lbKfE/PA66R1At0S/pKc6rWNsq08bPAG/PztwL+PjemTBsfDcyQ1AHMATw/fGw0FPtaYp6+pNcATwA/ImfyA95XTOxTo8yZJbpFLCvZxlcDnwY25013RMT9za5rKyvTzlXleyOip3k1bH0lv8tHAneRukI7gfdGxI4ah7MaSrbxbOBuUhf/z4FLImL3Iahuy6v8DuR7fU6LiNWFfScBDwEbgbNJse+VQY/VCkEf+u9inAc8nrvkhlXGBuf2aw6389hzG489t/H/j5y2/h3Ao/Uudlsm6JuZmdnItMqYvpmZmY2Qg76Zmdk44aBvZmNOUockHep6mI13DvpmbULSAklnD7KvayzXSpA0X9JNhde3SXq0UOTjwPo8favM8S7PSaDMbBRNPNQVMLNRswL4saSVpHnRFcuAacBCSUFK4HFNRHwZQNJM4GXqz6PuALqA30XE/qp9/wGWSzouIj4G7AP25uNfCHwUeH/1VKKcDncisC8iDhR2fRDYA7y7ULYDOAw4EBH76tTVzGpw0DdrA5JOBGYA7wFmA3MjolfS10gBdQmwJJftJWXxqthECtLFoHsYKcAXc35PyNtPJiezkTQJEPAL4CJgVY0FP24Ero6IysqMEyLi5bzvMmA1sFdSJZB3kk5A+iRtKxynEziCtBDRp0o1jJkdxEHfrD0sIq20tSNfzdfTf8UdEZOqd0paDNwaEdPqHOezwPVV2/pPFAp1OVfS3fn5g8CC/Py+/OcjEfHP/J77gMm5TDfwy4g4kFdyu4iUBtrMhsFj+mYtTtJE4EOkq/WKx3LAXQR0Sdoo6UVJu0hJPIZaqrMRtwLHAJ0RIWA68DzwDCm4nwp8nzS8MIHUe7Cw8P7DgTOBZyTNk/QAcDwpb/ty4CfABXmd8KeAE/IxzGwYfKVv1vquJI3TF82NiN7KC0mrSN33+6JGRi5J1wN7IuLORj64uChQThH6jfx4kbSozW7SCcYW4KaIWFv1/t3ARyStBfYDl5J6CjaQTgguiYgf5PsOFkXEukbqZ2YH85W+WQvLY/mfAdYMsr9L0nGkpWQXAoskLZZ0RlXRecA7q7ZNkDS58DhW0gk1PuMtuUt+PXBbRNxIGvufFBF/yce+Bbhd0vpcn2ovAbtI9yWcQ7pP4Arg9ZJmkdZmn+5pf2Yj46Bv1tp2kgLqr6q2V7r39wJTgc8Dl5PGyVcCb6oq30dhnD+bCvy78PgH8HCxgKS3Ab8m3WDXHRGrCvXaJakjkpWkhVlOpOp3R9IFwJPAuaRegQdJd+9/B5hPWhhnGmkWwgOSjqjXKGZWm4O+WQuLiL7iilsFc/MY++GkoPwQsDoiFpC60Z8qcfjnIkKVB+nu+eo8AFuA0yPi4oj4Y9W+2RRmBETExrytf+aApOXAPaSeiK2k1diOBL5ImgY4BzgdmEVaAvfNpDv+zWwYPKZv1qZyV3ilO3wjcL6k35LG7rc3eryI6CP1CBQ9DLxriF73A4Psk6QJwDeBb0XEs3njHNJ64F8F/hQRS/MSrdsj4nlJ3UDkHoRBlw81s9oc9M3a02OF528AvkvKihfA7aP4ORfmY/Yn15F0MvAbYAewPiJuqBTO8/QrUwRnkIYl9kuqTvbzKtIJw+LCe2EgV8B80p39ZtYAB32z9lRJztMJ9EVESPohaaz8+NH6kIjYU3yd1/X+OqnL/hPAz3KPw7KI2Jsz+e3P793MIL9BktYB2yJi6WjV1cw8pm/WLjoY+P/cWdkYEf8FXi1pBXA+sBm4S9KUvAhOt6RTSVP+jpJ0iqRTSPPhOyuv8+O0XH569YdLOkbSDaQr/K3AdRGxEziLNBa/RdK1ko4auyYws3p8pW/WHroYSFrTn2EvL8DzCOmO+Fmku/C/ADxNuinuSQ7Ou7+p6rjVrzvz8S7Nx19Kmgo4E/g98OGI+F6lcB6H7wGuIiXyWSnp/oi4os7fp3gSY2ajRDXydJhZG5E0JSL+VrXt2Era2xEe++3AecC63F0/VNlJpCmDOyPiiTplNwB/joirRlpHMxvgoG9mZjZOuPvMzMxsnHDQNzMzGycc9M3MzMYJB30zM7NxwkHfzMxsnHDQNzMzGyf+B7R+4OMS/H1LAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 绘制真正类率与假正类率的曲线图，趋近于左上角的（0,1）的切点为最优点\n",
    "\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_roc_curve(fpr,tpr,label=None):\n",
    "    plt.plot(fpr,tpr,linewidth=2,label=label)\n",
    "    plt.plot([0,1],[0,1],'k--')        # 绘制随机分类情况下的曲线\n",
    "    plt.axis([0,1,0,1])          \n",
    "    plt.xlabel('假正类率',fontsize=16)\n",
    "    plt.ylabel('真正类率',fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "\n",
    "plot_roc_curve(fpr,tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9618627478983288"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算曲线下面积AUC，从AUC角度来查看分类器的分数,AUC的取值范围在[0.5,1],越趋近于1，则效果越好.\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_train_5,y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   * 召回率TPR越高，分类器的假正类FPR就越多\n",
    "   * 虚线表示纯随机分类器的的ROC曲线，好的分类器应该远离这条线，向左上角趋近\n",
    "   * 是使用精度/召回率PR曲线 还是使用ROC 曲线，关键在于 正类非常少或者更关注假正类而不是假负类，则选择PR,反之ROC\n",
    "   * 上述例子 正类数据真的很少，PR的提升空间较大，所以选择PR更合适，ROC曲线表现不错是因为跟负类非5相比，正类真的很少"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练随机森林分类器，比较SGD分类器的ROC曲线和ROC AUC分数\n",
    "   * 获取训练集中每个实例的分数\n",
    "   * 随机森林分类器没有decision_function(),但是拥有dict_proba()方法，sklearn 中分类器都有这两个中的一个\n",
    "   * dict_proba()返回每个实例属于正类或负类的概率，\n",
    "   * dict_proba()返回一个矩阵，每行一个实例，每列代表一个类别的概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "# 初始化对象\n",
    "forest_clf = RandomForestClassifier(n_estimators=10,random_state=42)        # n_estimators=10设置10颗树\n",
    "# 通过3次折叠，使用predict_proba方法对训练集实例进行交叉预测，得到实例的分类概率\n",
    "y_probas_forest = cross_val_predict(forest_clf,X_train,y_train_5,cv=3,method=\"predict_proba\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       ...,\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probas_forest    # 二元分类，返回两列，每行的概率和为1，左为负类概率，右为正类概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制ROC曲线，需要决策值而不是概率，直接使用正类的概率作为决策值\n",
    "\n",
    "y_scores_forest = y_probas_forest[:,1]\n",
    "fpr_forest,tpr_forest,thresholds = roc_curve(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde3zP5f/H8ce12WbmLHOqfCk6oIkhEnKKzjoh5yLVT307l1IkSenbufR1KCSpvp1TckyksNGopAM5xgjDDDtcvz+urQ2bbWyf9/be83677fv57P15f97v53zXXp/rel/v6zLWWkRERMT/grwOICIiIoGhoi8iIlJCqOiLiIiUECr6IiIiJYSKvoiISAmhoi8iIlJCqOiLiIiUEAEv+saYasaYRcd5PcQY85kx5ltjzE2BzCYiIuJnAS36xphKwBQg4ji73QHEWmsvBK4zxpQLSDgRERGfC3RLPxXoDuw9zj7tgPfSn38DRBdyJhERkRKhVCBPZq3dC2CMOd5uEcCW9Oe7gGpH72CMuQW4BSAiIqLp2WefXbBBpdiy//xPxkP6/x6xzT3JeP3Y95DlPVmOcsw+9tjjZjlX7sfOss8RmbLmtNm+78htWRMde6y87GOzhMztZ8n+9cxjHLkt67mOypDt/yfH7nPE+bJs0ATiArhfBAMZVcX9/hmC0pu0BkhLc1+lSoExYDCkpkFqKgQHQ6lgt6NNg8OH3T6lwzIPmnTAHTciwr0GcOigITkFSpeG0BC37fBhOHgQQkIgoow7e2oq7NvnzlO+fGbOv3cB1lClSnomA3v2uJ+nYsX08xhIOuCOERZ6mIQdm0k+mER4uUok7du901pbNb//XAEt+nm0HwgHEoCy6d8fwVo7HhgPEB0dbWNiYgIasCg6mJxK/N5DbN93kG0JB9m+9yA79x8mNS2NNAtp1hWoNGvTv1whSEvjyO/Tn+e4v81m/7R87p/d8dPyuf9Rxy8pS0iYox7FMQaCjCHIuEZFkIFgYwgyxr0WZI55PSjr61m2mexeC8p4Let783KsLK8HZeyf3bmO3D/5sGH/fihfzlAm3G1LSDBs2giVKxvq/MvtfzDJ8MUXcPpphgtbZR5/9ldwMMlw2WX88/558wy//27o0hnq13fbYmIMX8w0tGgOV17h8iTsMdx2q+Gs+jB6dGa2O+6A+HjDf183VIt02556CubOMQx7xNC1izv/vDmGYcOgU0fD00+7/XbuNHS4GE6pYli8OPPf68or4cfVhpmfQ+PG7vyPDTNMmGB4chTcdpsBC5MmGu6719C7N0yc4DKtW+cy1q1r+OOPzN+FGjVg2zbYsgVq1nTbevSAd9+FYY/D3XdDuXLw+uvu+UUXwZdfuqK8di1cdhmcdRbMnJl5zEsugcREmDULypZ120aPhrlzYehQ6NTJbVu4EB5/HBo0gJdfdtt27YJnnoGLL3bHybByJVStCqeemvvvd2JiInXr1iU8NIS3Jr9J9+7dMcZsOJH/Vopi0Y8FWgP/A6KA772N463UNMvfiYfYnnCI7XsPsm3vQeLTH7ftPfTP8z0Hkr2O6rlc/5B7/Ic/OCg/hSGPhcQcVUiC8rl/nn++7P9tjvj5g/K5/zFFOW/7Y+HQYUPZMm7bhg2wdq2hTBlo0ybz92H+fNfC6tAh8w91XBz8+itERUH9+m7b77/DZ59B3bpw1VVuW2IijBwJZcrA8OGZx3zpJVi/Hu68E+rUgR07YMYM+O47uKEHXHml2+/jj6FbN7jjDvcecFnq13dZfvst85hXXgnffusytGrltjVrBjExMH48DLrBbXvsMXj3CbjtNhjU3bUGV66EYeNhXXV4Z1jmMUf1gq1bYeL9UKuW2zbrZfj+XbizK/Rs7rbtXAY/z4S2p8PV57ttGzfCgbXw858Q9V9XRAGS/oBdGyC6FvzrX25bpAWzA+qUhaa13bYNlaHMQYgMgQbpRXdHKah7iitydbO0Tc85HUiEU6tAtfJuW5MG0OViOLsuVAh32zq3h+uvcV+h6VWrZg145x1XwLP6+GPXgq9SJXPba6/BCy+4f7PSpd22W291X1mddZb7fTjaV18du+3hh91XVm3but+7rCpXhjFjjn3/+ecfu+1oycnJhISEEBERwcsvv0yzZs2oU6dO7m88DuPFKnvGmK+tte2MMe2Bc621r2R5rTbwBTAXaAVcYK1NzelYxbWlb61l36EUV7SPU9Dj9x0iNS33/49KBRmqlS9NtfJh6Y+lqVoujJBgk+Mf8uAT+SN93NZKLkUw6AQKScYxgo4uZMfuL0VLcjKsW+da4RkFFmDqVEhKgv79ISwMDh2C++6D/fvhwQch42rdp5/C//0fPP003Hij2zZ/PnTpAtddB9Onu+7UK6+EzZvd819/zTxPvXruD/jatZnnv+ceeP55ePZZuPdety2jQF95JXzyidv2999wyilQqZJrqWVo1coV+MWL4cILYfduGDcOHnnEtfyGDnX7ff45XHGFK47r17tte/dChQqu6O/bl3nMtm3hm29gwQJo185te/JJGDYMpk2DXr3ctkWL4Prr4eab3b9T+fIu21NPuUL2xBOZx3zzTThwAPr1y/zAExPjPgg0bZr5QSA+HjZtcsX49NMz/3/79VcID3cfhDIkJLjHcuX4p+tcCs/q1avp0aMHI0aM4Prrrz/mdWNMrLU232PePGnpW2vbpT/OB+Yf9doGY0wnXGv/seMV/KLqcEoa8ftcF3tGQc/4coX9ENv2HuTA4bz9aJUjQqlWvjTVsxT0auVLU71CGJHlSlO9QmkqlwklKEiFT05ORhvAGEhJcYUoNhauvdYVUXCt0uefd12agwe7bT/8AIMGQcOGruAA/PWXK+CnnuoKS4Z//9tdu7z+elf0Dx6EP/90hbJfv8yi/+uvrph//rnbNyTE5UtOziyaoaHwxRfw0EOwc+eRP0uHDnDOOe46bIaoqCN/FoAzz3SZGjXK3FamjCum4eFHHvPOO90HjoyWbqVK7oPAuHGumzjDRRfBsmVQu3bmtrJlXZfz0QXzk0/cv3WFCpnbHnnEfWV10UWu2zqrypVh7FiOMWDAsduisykPkZHuK6uQENc9fbSs+aTwWGt5/fXXueeee6hYsSKVK1cu0ON70tIvSIFs6aelWXYdOJxZwI8p6K51/nfi4TwdLzwkmOoVXOu8epZinrWgR5YPI6xUcCH/ZFIc7dvnCl1Gb9+SJfDhh674nnWW2zZ/visA7dtnFmNw1zrT0lxhNsZ1h9as6Vp+33zjCswnn8Cjj7ri+5//uFY3wH//67pF777bdV/XqQPffw8tW0KLFu45uGO1bu26hxcuzDz3v//tWvr/+U9m1+z//ufOM2BAZnfy2rWwapXLdeGFbtvBg67FWe2Y4b0ixd+uXbsYNGgQH374IV26dGHKlClEHv2pLF2xaukXVfH7DvJ7/P5sC/r2vYeI33eQ5NTcPyQFBxkiy4UReVTrvHrWgl6+NOXCSqlbuoRLSXEDiDJ+Dd55x13v7ds3szX5v/+51u6oUZmDfj76CK65xl3nffhh12KOi4MpU9z16G7doHNnV1w3boTt2488719/uUdr0wfBBUHz5u48yenDQ666Cs47z3WJZ20tt2rlWvq1a2cW34YNXcu2fPnM/SIjj+xuz/Dii8duu+66Y7eddVbmh5cMpUtnXpMV8Zu5c+fy6aef8uyzz3L33XcTVAjXUVT0gb0Hk3lp7m9MXvInKblcP69YJoTq5UvnWNCrVQijSkQYwepqLzGSkjK7gQ8edK3t8uXh8svdtp9+gokT3TXr1193RR4yC/26dZmt9XfecQO6Lroos+ivXg3z5rlrtHfc4V7L6KCbO9d1idet61raDz7oWs8tWrjXL77YXVMuU+bIzFu2ZN4mlJFl+nTXBZ31c2idOu4DRlaNGh3ZFQ7ufc2a5fufTqTES01NJS4ujiZNmnDDDTcQHR1N3ayDKQpYie7eT0uzvB+7ibFfrWXn/sMYA01Pr0SNiuHHXj8v77raS4eoq91PNm92Xw0aZHY1P/44rFjhRm03aeK2ffopvPqqG5w1ZIjb9s03bhBW586Zo3s3b4bTTnNfGze6bTt3uoFS4D4gZLRUmzeH5cvdh4Jzz3XbXnrJddM/+SSccYbb9sMPrvBXqeKuo4eEuOPs3595XBEpfrZs2ULv3r35/vvvWbt2LadnjKbMA3Xv51Pshl2M+PRnVm9xQ1Kja1di+BUNaHSqRqsUN4mJbhTzaae573fvdrfnlCrlBkMFBbnW7j33uH3HjXMF9ccfXat5zx43Mrp1a/f+775zRTzr7TybNsHs2W7QV4aMFntious+r1bNDexq186NSM9wyiluoFndupnvAddKj4g4ctudd7qvrBo3dl9ZhYcfO8hMRIqPzz//nP79+5OUlMS4ceM4LeMPWCErcS397XsPMubLX/hopZv0r3r50gy99GyujKqp6+tFyIEDrnUbHJw56njePNdNHhmZeV34u+/cNebq1TOvU2/e7EZp79rlbuMKCXGt7R493DGWLHFd4QADB8KkSe56dEb39JIl7v7rCy7IvGa9cSOsWeM+WGS0ylNSXHd+6dLuA4aISG6stdxzzz288MILNG7cmBkzZnDW0YNX8uBEW/olpugfTE5l0uL1vLrgdw4cTiW0VBC3XFSX29qdQUSY/mIHUlqaa41nTJ6xZInrUm/VKnMilKVLXdHt0sWNIi9Vyl37rlfPtbYzJjexNvMWqKy/yoMGuWL/0kuZBfmtt9yI9169dPuRiHjnvvvuIzk5maeffprSJzgyVd37ObDWMufn7YyauYaNuw4A0Pncagy77FxOr1Iml3dLfiQnu5Z1xkjvRYtci7xVK9e1Dpm3dnXq5Ka0DApyLeXZs13r/p573LX1ihXdYLGDB11XeWiomzzkySePnMnKGPe+o7u6J0w4Nl+fPoXzc4uIHI+1lqlTp1KvXj1atWrF2LFjPetZ9nXR/z1+P49/9hOLfnOzdtSLLMvwKxrQut4pHicr3hIT3bSSLVvCpZe6bRkzkPXrB5Mnu21bt8IHHxw5GUmlSu5xzhx3nbx2bTdbWq9e0LVr5uxhZ53l7tHOKjT02GkvQde2RaTo2rdvH7fddhtvv/02ffr0oVWrVp5eSvZt0d97MJnrX1/C7gPJlC9dirs71af3BbUJCdb8kfnxyy+uiEdEuIlawM01PmqUm7a0ZUtXyDMK75Ytme9t1Qrefz9zek9wXfO7d7uWfIayZd10oyIifhITE0OPHj1Yv349TzzxBEMz5mn2kG+L/nvLN7H7QDJRp1bgjf7NqFI2zOtIRdLOnW50Obju+dtvhz/+yFw0Ys8eN/d5/fqZRb9tW9fFvmFDZsu8dWt3rKyLXGTcupZVcPCRBV9ExI+WLVtG69atqV69OgsXLqR1xu1BHvNl0U9JTePNb/8E4I729VTwcYPc/v7bzdUdFORmb4uOdq30+Hi3T1CQGx0fFOQGzdWt6+4l79TJzRCX4cwz3X3sWYWFuS8RkZIsLS2NoKAgmjZtyrBhwxgyZEiBz59/MnzZ1z3rp21s2ZNEnVMiaH929vMW+9mUKW7FrYxinpzsCnnVqpmLkjRs6G4527Ejc9R7cLBbyvONNzJb8EFBbpBd796B/zlERIqTuXPn0rhxY7Zu3UpwcDCPPfZYkSr44NOiP2mxW8vyptZ1SsTKc4sWuVXKMjz/vJuTPeO2tlKlMlcAmzPHPQYHu/vZU1KOnHZ10CA3GC+HNR5EROQoycnJDB06lM6dO5OSksLevXu9jpQj3xX92A27WblxDxXCQ7i2SS2v4xS6W26BNm1c6z4pyW3r3dtdd8+4nm4MfPmluz8+Y21ucGtqB2tWYRGRE7Z+/XouuugixowZw8CBA4mJieHsjPWhiyDfFf1Ji9cB0KvF6ZQJ9deQhaQkaNrUTVqTsWraE0+4xwkT3D3tAPfdB19/feSo+YiII1v0IiJy8kaOHMkvv/zCu+++y/jx4ylz9OpWRYyviv6mXQeY9eM2QoIN/Vr9y+s4J+33393yqZ984r7fudOtNb50qVuEBdw0sbt2ua76jHvgRUSk8Bw4cIDNmzcD8Pzzz7Ny5UpuuOEGj1Plja+awpOX/EmahavOq0m18sVz0e2M9c0PH3at+r173W1zV13luut79nSj5zOu0YOKvYhIoKxevZru3bsTERHB0qVLqVixIhWL0X3Ivmnp7z2YzLvLNwFwc+s6Hqc5MU895eaL37fPzT63Z4/blnVlt1693ApyRbwHSUTEV6y1jBs3jmbNmrF7926eeuopgoKKXwktfolzsOCXePYfSqH5vyrTsFbxWE3FWjc7Hbjr8Zs2QWoq/O9/bpsxbknW8eO9yygiUtIlJCRw7bXXcvvtt3PxxRcTFxdHx44dvY51QnxT9H+P3w9Ai7pF657InPz2m7sH/oor3G1zpUtnzj/fo4fX6UREJENoaCgbNmzg2WefZebMmUQW43uafVf0z4ws63GSnCUluVnxAM44w01/++23bn57gAsvhC++0AIyIiJeS01N5cUXX2Tfvn2Eh4ezdOlS7r333mLZpZ9V8U6fRUbRP6Nq0Sz6r7zirsNPmOBa9kFBrsAnJ7vZ8UREpGjYvHkzHTp04K677mL69OkAlCrlj3Hvvij6Kalp/Pl3IgB1q0Z4nCZ711/vHocOdQP0AJo1c7PliYhI0fDZZ5/RuHFjYmJimDx5MrfccovXkQqUL4r+xl0HSE611KoYXqQm5Pn5Z7eePLj76d95x82Kl7GqnYiIFB2vvvoqV155JaeffjqxsbH069cP47NZzYpOhTwJ/3TtF6Hr+StXQpMm0LEjdOniZsTTAD0RkaLr8ssvZ+PGjYwcOZIwny4b6ouW/u870gfxeXw9f/JkmDnTPW/c2F23X7Eic8pcEREpOqy1TJ48me7du5OWlkbt2rV5+umnfVvwwSdF/494dz3/jEjvrucPHAgDBsB777klbY2BAwfcaP26dT2LJSIi2di7dy+9e/dmwIABxMfHs3//fq8jBYQ/uveLQEu/d2+3HO0XX7iufAAff1gUESm2li9fTs+ePfnzzz954oknGDp0KMElZMlRXxT9Pzy6R3/cONeKv+QSaNcOWrWC0aMDGkFERPIhOTmZG264gbS0NBYuXMiFF17odaSAKvZFPznVsv9QChXLhFA5IjRg5/3oI7j9dvd80CA3VW5o4E4vIiL5sGPHDipVqkRISAgfffQRtWvXplIJXK2s2F/TT05NBaB2lYhCv7UiMdEtYQvQrZt7ft557t57EREpmubMmUOjRo0YOXIkAI0bNy6RBR98UPTTrHssG1a412MOHHBd+Vde6Va5A6hVC+LioE7xXNRPRMTXkpOTeeihh7jkkkuoXLky12fMklaCFf+in171w0MK90pFmTJuhP7KlVCuXKGeSkRETtL69eu56KKLePrppxk4cCAxMTE0atTI61ieK/bX9DNa+uGhhdPST01199sbAzffDP/+txulLyIiRdfu3btZv3497733nlr4WRT/lr7NaOkX/I+yaROceSa88Qbs3eu691XwRUSKpsTERN5++20AmjRpwvr161Xwj+Kbol8Yc+5v3gw7d7pu/YwBfCIiUvSsWrWK6Oho+vTpw08//QRAmTJlPE5V9BT7op9e8ykdUvDd+y1bwpw5sHYtnHtugR9eREROkrWW1157jebNm7Nnzx7mzJlDgwYNvI5VZPngmr4lGAgvwKI/Zw7UrAkNGsAFFxTYYUVEpID179+fqVOn0rVrVyZPnkykrsEeV7Fv6WcM5CtTQAP5tm+Hzp3h3XfVpS8iUtR17tyZ//znP3z++ecq+Hngi5Y+QOkCKvrGQNu27l784cML5JAiIlJAUlNTefLJJ6lRowaDBg2iV69eXkcqVnzQ0s8YvX9yRT8tzT1GRsKQIfDNN1BC1l8QESkWNm/eTIcOHRg+fDjLly/3Ok6xVOyLfsZAvpMt+jfeCLGx7vl110HjxicZTERECsynn35KVFQUMTExTJkyhfHjx3sdqVgq/t37aRm37J140f/yS3cNf80aWLZMS+KKiBQla9as4eqrr6Zx48bMmDGD+vXrex2p2Cr+Rb8Abtlr2BBefRXWr1fBFxEpKhISEqhQoQLnnHMOH330EV26dCFMf6RPSrHv3v/nmv5JtPRPOw169IAxYwoqlYiInChrLZMnT6Z27dosWbIEgKuuukoFvwD4puifSPf+gQOZt+VVrqyBeyIiXtu7dy+9evViwIABnH/++dSuXdvrSL7ig6LvHk9kIN9zz8G//gW640NExHvLly/n/PPP57333mPUqFHMnTuXWrVqeR3LV4r9NX2bcZ/+CRT9HTugfn245pqCTiUiIvk1Z84cUlJSWLhwIRdeeKHXcXzJZBTN4iq8Zn1bre/z/PT4JUSE5f8zTHIylCrlJuUREZHA2r59O+vWraNly5akpqayb98+Klas6HWsIs8YE2utjc7v+4p9Sz/tJFr6ACEhBZlGRETyavbs2fTt25fQ0FB+//13QkNDVfALWbG/pg8QGhxEcFDem+qHD7spdmfPzpyJT0REAiM5OZmHHnqISy65hCpVqjBz5kxCQ0O9jlUiFPuWPkBIcP765qdOhZEj3SC+desKJ5OIiBxr7969dO7cmaVLl3LLLbfw/PPPa937APJF0c9PKx+gTRs3cr9aNV3LFxEJpHLlytGwYUPuvfderr/+eq/jlDglsujXr+++RESk8CUmJvLAAw9w1113Ua9ePSZOnOh1pBLLF9f0g4Py/mP8/HMhBhERkSOsWrWK6Ohoxo0bx/z5872OU+L5pOjnbb8NG6BBA5g5E1JTCzeTiEhJZq3l1VdfpXnz5iQkJDB37lwGDx7sdawSL+BF3xgzyRjznTFmWA6vVzLGfGGMiTHG/DcvxwzO44X5cePcY7dumnJXRKQw/fe//2XIkCF06NCBuLg42rdv73UkIcBF3xhzDRBsrW0J1DXG1Mtmtz7A2+mTDpQzxuQ6+UBwHkfvjxnjls9dsCA/qUVEJK8OHToEQL9+/Zg0aRKff/45VatW9TiVZAh0S78d8F7689lA62z2+RtoaIypCJwGbMrtoHlt6QOcfTZodkcRkYKVmprK448/TuPGjdm3bx/h4eHcdNNNGN0iVaQEuuhHAFvSn+8CqmWzz2KgNnAnsCZ9vyMYY25J7/6PgbyN3l+1ChITTzC1iIjkaPPmzbRv354RI0YQHZ3vmWElgAJd9PcD4enPy+Zw/uHArdbakcAvwICjd7DWjrfWRmfMO5yXon/ffVC2LMyYcaLRRUTkaJ988glRUVHExsYyZcoU3nrrLcqVK+d1LMlBoIt+LJld+lHAn9nsUwloZIwJBloAua4IlJdb9hIS3GPjxnnKKSIiuUhLS2Ps2LHUrl2bFStW0LdvX68jSS4CPTnPx8AiY0xNoCvQwxgzylqbdST/U8CbuC7+74B3cjtoXm7ZW7oUDh0CTe8sInJyfvnlF6pUqULVqlX58MMPqVChAmFhYV7HkjwIaEvfWrsXN5jve+Bia23cUQUfa+0ya20Da21Za20na+3+3I6b18l5wsI07a6IyImy1vLGG2/QtGlT7rnnHgAiIyNV8IuRgN+nb63dba19z1q7raCOmdsde7NmuZX1RETkxCQkJHDjjTdy880306JFC55++mmvI8kJ8MWMfKVyael37epa+b/9FqBAIiI+8tNPP9GkSRPef/99Ro0axZw5c6hZs6bXseQE+KLoH6/mHzwI6b1Q1K0bmDwiIn4SGRlJtWrV+Oabb3jkkUcI1pSmxZYviv7xWvqlS8Mzz8Bbb2nqXRGRvNq+fTsPPPAAKSkpVK1alW+//ZZWrVp5HUtOki+KflAu9+kHB0Pv3gEKIyJSzM2ePZvzzjuPl19+mRUrVgBoZj2f8EXRL5VD0bcWVq92jyIicnyHDx/mgQce4JJLLqFq1aosX76c5s2bex1LCpAvin5QDp9AN2yA886DOnUCHEhEpBgaMGAAY8eOZfDgwSxbtoyGDRt6HUkKWKAn5ykUObX0v/0WqlVz0++KiEj2UlNTCQ4O5t577+Waa67h2muv9TqSFBJfFP2c5t7v1cvdrrcp13X6RERKnsTERO68805CQ0MZN24cTZo0oUmTJl7HkkLkj+794wzkq1wZoqICGEZEpBiIi4sjOjqaN998k8qVK2M1+KlE8EXRz25GvtRUiI8PfBYRkaLMWssrr7xCixYtSEhIYO7cuTz55JManV9C+KLoZzeQb8kSdz3/rLM8CCQiUkRt2bKFoUOH0qFDB+Li4mjfvr3XkSSAfHFNP7tPqFu3QkQEVK/uQSARkSLmxx9/pEGDBpx66qksW7aMs88+W637EsgXLf3sfm+7d4cdO2DixMDnEREpKlJSUhgxYgRRUVFMmzYNgHPOOUcFv4TyRUs/p3F84eFQr15gs4iIFBWbNm2iV69eLFq0iD59+nD11Vd7HUk85ouWfnbX9H//HdLSPAgjIlIEzJw5k8aNG7NixQqmTp3K1KlTKVeunNexxGO+KPpH13xroWVL18r/6y9vMomIeK1OnTqsXLmSPn36eB1FigifFP0jq35CAuzcCevWaSCfiJQca9as4c033wTgsssuY+nSpdTTNU7Jwh9F/6jvK1aEAwdg48bsB/mJiPiJtZZJkyYRHR3NI488wv79+wG07r0cwxdFP7tr+uHhcNppHoQREQmghIQEevbsycCBA7nggguIiYmhrBYckRz4evS+iIifHTp0iObNm/PHH38wevRoHnjgAbXu5bh80dI/+pr+669DmzYwfbpHgUREClHGPPlhYWHcfffdfPPNNwwdOlQFX3Llk6J/5PeffQaLFkFMjDd5REQKy7Zt2+jatStffvklALfeeiutWrXyOJUUF74o+kdf03/+eXjmGbj0Uo8CiYgUgtmzZxMVFcXChQvZuXOn13GkGPLFNf2jL+nXrw933w2lfPHTiUhJd/jwYYYNG8bYsWNp0KAB8+fPp0GDBl7HkmLIHy39bEbyqeCLiF988sknjB07lltvvZXly5er4MsJ80XRz9q7v38/NHOrLhMAACAASURBVGoEX33lXR4RkYKwceNGAK677jq+/fZbxo0bR3h4uMeppDjzR9HP0sH/88/w449w000eBhIROQmJiYncdNNNNGjQgPXr12OM0WA9KRC+6ATP2rt/6qnw3HNuGl4RkeLmhx9+oEePHvz66688/PDDnKZZxqQA+aToZ1b9mjXdID4RkeLmlVde4d5776VKlSrMnTuX9u3bex1JfMYf3fuakU9EfGD16tV06tSJuLg4FXwpFL5o6WedkW/CBKhWDVq3hsqVPQwlIpIHCxcupHz58px//vm8/PLLhISEHDPLqEhB8UVLP+s1/WnT4KqrYN487/KIiOQmJSWF4cOH0759e4YNGwZAaGioCr4UKn+09LOM3m/d2t2jHx3tYSARkePYtGkTvXr1YtGiRfTt25dXXnnF60hSQvii6Gdt6T/5pHc5RERy8+OPP9KmTRuSk5N566236N27t9eRpATxRfe+esNEpLg4++yz6d69OytWrFDBl4DzSdF3VT85GTZvhoQEjwOJiGSxZs0aunbtys6dOylVqhTjxo2jXr16XseSEsgXRT/jPv316+G003Q9X0SKBmstkyZNIjo6mtjYWP744w+vI0kJ54uin9G9f+gQ1KrlJugREfFSQkICPXv2ZODAgbRs2ZK4uDhatGjhdSwp4Xw1kK9RI9e9LyLitfvvv5///e9/jB49mgcffJCgIF+0saSY80nR10g+EfFeWloaCQkJVKpUiSeffJIBAwbQsmVLr2OJ/MMXRT9DaiocPAgREV4nEZGSZtu2bfTt25cDBw7w9ddfU7VqVapWrep1LJEj+KK/KaOlP2gQNG0KU6d6HEhESpSvvvqKqKiofybbCQ4O9jqSSLZ8UfQzevdXrIC1a2HLFm/ziEjJcPjwYe6//366dOlCZGQkMTEx3HLLLZpKV4osX3TvZ/zntWQJLF4MZ57paRwRKSEOHTrExx9/zK233spzzz1HeHi415FEjssXRT9DmTLQsSNokKyIFKZPPvmEzp07U65cOWJjYylfvrzXkUTyxBflMWtXmgq+iBSW/fv3M2DAAK6++mpeffVVABV8KVZ81dIfMABCQmDcONA4GhEpSD/88APdu3fnt99+49FHH+Wuu+7yOpJIvvmm6KemwuTJblDff//rdRoR8ZN3332Xvn37csoppzBv3jwuvvhiryOJnBBfFH1jwFp3q94vv2jVPREpWE2aNKFbt2688sornHLKKV7HETlhvij6AKVKQe/esGeP10lExA8WLlzIBx98wIsvvki9evWYMWOG15FETpqvhr0ZA5UqeZ1CRIqzlJQUhg8fTvv27Zk1axZ///2315FECowvir4B4uLgmWdg5Uqv04hIcbVp0yYuvvhiRo4cSZ8+fVixYoW688VXfNO9v2ABPPggLFoEn33mdRoRKW5SU1Pp2LEjW7duZdq0afTq1cvrSCIFzjdFv2FD6NYNWrXyOomIFCcHDx4kJCSE4OBgxo8fT61atThT03qKT/miex9j6NgRPvwQ7rvP6zAiUlysWbOG5s2bM3bsWADatm2rgi++5o+in85arxOISHFgrWXixIk0bdqUbdu2ERUV5XUkkYAIeNE3xkwyxnxnjBmWy36vGWOuyNMxcdfyN22ClJQCiSkiPpWQkEDPnj0ZNGgQrVq1Ii4ujq5du3odSyQgAlr0jTHXAMHW2pZAXWNMvRz2uwiobq3N85C8F16A2rVh/PgCCisivvTzzz/z8ccfM3r0aGbPnk2NGjW8jiQSMIFu6bcD3kt/PhtoffQOxpgQYALwpzHmqrweOCwMqleHc84piJgi4idpaWnMnz8fgJYtW/Lnn38ydOhQgrRCl5Qwgf6NjwC2pD/fBVTLZp++wM/AM0BzY8wdR+9gjLnFGBNjjIlx38P06fDXX9C2bSElF5Fi6a+//qJz58506NCB2NhYAKpXr+5xKhFvBLro7wfC05+XzeH85wPjrbXbgGnAMStbWGvHW2ujrbXRR7+mD+4ikmHWrFlERUWxZMkSJkyYQJMmTbyOJOKpQJfIWDK79KOAP7PZ53egbvrzaGBDXg68ejWkpZ1sPBHxi2HDhtG1a1eqV69OTEwMAwcOxGg1LinhTrroG2OC0gfe5cXHQB9jzHPADcBPxphRR+0zCbjYGPMNcDvwbK4ZMNx+u7umv3VrftKLiF+ddtpp3H777SxdupRzzz3X6zgiRUKuM/IZY0KBe4ExQGlrbVL69tJAd9zAvK+AMrkdy1q71xjTDugEPJPehR931D77gOvz92PAmjXw998QEpLfd4qIX0yfPp3g4GC6d+/O4MGDvY4jUuTkpaUfBNwP3AE8lmX7NOBh3G3yyXk9obV2t7X2vfSCXyCMcffpL1+uVfZESqL9+/fTv39/evXqxZQpU7CaqUskW3mZe/8wkAh8AcQYY74D6uFuv2tqrT1gjEktvIh5o1v1REqmlStX0qNHD3777TceffRRHnvsMV27F8lBrkXfWptmjEm21v5ujLkb2AisBJYBVxlj3jv+EURECse6deu44IILqFq1KvPnz6ddu3ZeRxIp0vI7kG+btfYH4BTgJWAscFqBp8qng0nQvj0884zXSUQkEFLS59uuW7cuL774Ij/88IMKvkge5LnoG2OaAx8YY7rgbqVbB2y31i7HXdf3TGIiLFgADz7oZQoRCYSvv/6a+vXrs3LlSgBuvfVWTjnlFI9TiRQPx+3eN8ZcAAxN/3YlrmX/MdALN8I+If32u/D0R3AfJEpba28tnMjHCi8Dzz4LZcsG6owiEmgpKSmMHDmSUaNGUa9ePU2hK3ICcrumXxc3dW4I8BEwAvg37l56C+wFzsAV+jrp7wkGShdC1hyVjYCb7w3kGUUkkDZu3EivXr1YvHgx/fv35+WXX6asPuWL5Ntxi761djow3RizGVfgn8YV+w7AJ7h7828GfrPWdivkrCJSQr355pvExcUxbdo0evXq5XUckWIrr/1jh621NwK7gQrAQeA6oDxQG/dBwDOJiYZ589xUvCLiD0lJSfz8888APPzww6xatUoFX+Qk5fei2OvAOcDfuK7/aGttbIGnyqf166BjR3jgAa+TiEhB+Pnnn2nRogWdO3cmKSmJkJAQ/vWvf3kdS6TYy7XoGzfLRZgxpjIwA3d9PwJ3y15k4cbLm/AycPHFEBXldRIRORnWWsaPH090dDTbt29n4sSJhIeH5/5GEcmTvMzIF4a7dt8FeMda+yOAMaYvMNUY0woILbyIuTvzTHh4vpcJRORkJSUl0a9fP95//306duzIW2+9pXXvRQpYXrr3U4AhuFb+QxkbrbVfAi8AabgPBiIiJywsLIxDhw4xZswYvvrqKxV8kUKQl2l4U4C3079NPOq1p9K7/5sWQrY8S0sFa93COyJSfKSlpfHcc89xww03cPrpp/Pxxx9r3nyRQnTSs1tYZ1VBhDlRH3wAQUEwerSXKUQkP/766y86d+7M/fffz5QpUwBU8EUKWZ6KvjEmzBjzoTEmLP37U4wxkcaYCGNMqjEmIsu+U40xFxZW4Ozs3h3Is4nIyfryyy+JiopiyZIlTJgwgWHDhnkdSaREOG7RN5kfu9OAq9IfAd4AvgKScfPuH0rfvzzQA6hZGGFzMngwbNkCAwYE8qwiciJmzJjBpZdeSvXq1YmJiWHgwIFq4YsESG4t/U+MMVdaa5MBrLXJxphBuJH891prD7vNNiV9/764CXw+LrTE2TDGULMm1KgRyLOKSH5Y6+bwuuyyy3jsscdYunQp5557rsepREqWHIu+MSYIt8jOO+m352GMOQ34D/CAtXb+UfuXBu4Chmd8SBARAZg2bRqtW7cmKSmJcuXK8fjjj+v+exEP5Fj0rbVp1trhuNX0+qRvfglYaq19IZu3PAX8BYwv8JS5ePMN6NoVfvop0GcWkePZv38//fv3p0+fPgQFBbFv3z6vI4mUaLkO5LPWfmGt7YS7dv8g0A+OuN5vjDH/Aa4GrrPWpmV/pMKzfDnMmqUBfSJFycqVK2natClvvfUWjz32GAsWLCAyskhM4ilSYh33Pn1jzCzgQPq3FhgDBKWP4t9jjGme/toVQEtr7fZCS3oc990HaetAlwdFigZrLbfffjuJiYnMnz+ftm3beh1JRMh9cp4VpI/Mx7XkzwHexU27uxVYArwInAo8Zoz5txfX8888E665IdBnFZGj7dy5k1KlSlGxYkWmT59OuXLlOOWUU7yOJSLpjtu9b6192Fr7OG7wHrildMumb3/FWvsyrgegMdAMmFCoaXNw+LCbkU9EvLNgwQLOO+88hgwZAkCdOnVU8EWKmLyssvcUMBdX3C8CehljhmTdx1r7K+4+/q7GmCsLI+jxvPMOjBsHBw7kvq+IFKyUlBQeffRROnToQPny5bnvvvu8jiQiOchtcp57gIHAvwGsteuAXsBTxpi6Gbulv7YVd81/eKGlzcE3C+H//k+tfZFA27RpE23btmXUqFH079+f2NhYGjdu7HUsEclBbi39H4HLgWXg7t1Pvz//c+DZbPafAjQ0xjQs0JS56HwJDBkCuu1XJLCCgoLYtm0b06dP54033iAiIiL3N4mIZ447kM9aOxvc3Pu4Fn15YA+uRR9rjDkr43Vr7SFr7S5jzEqgG+4DQ0D07GG4+vxAnU2kZEtKSmLChAkMGTKEWrVq8csvvxASEuJ1LBHJg7yusmdxo/TTAKy1ccAFwAZgIeld/OneAeYVYEYRKSJ++uknmjdvzr///W++/vprABV8kWIkT0XfWnvYWnu3tXZvlm0x1tqD1tqLrbUHs2x/0Vq7pDDC5mTLFli/PpBnFClZrLWMHz+eZs2aER8fz6xZs2jfvr3XsUQkn/La0i/SHnwQNPeHSOG5++67GTx4MBdeeCFxcXFccsklXkcSkROQ2+Q8GGNKATWstZvysO8ZwBhr7fUFES6vataEU1MDeUaRkuX666+nRo0a3H///QQF+aKtIFIiGZvLfW7GmCbAYmttmSzbqgNfAK2ydu0bY6LS9y1XSHmPEVajnn33i6+5+vxagTqliO+lpqby9NNPs3fvXsaMGeN1HBE5ijEm1lobnd/35eUj+0Hg6Kl1k4Eo4PBR2w9ns6+IFCNbt26lc+fOPPLII2zYsIG0tICvoSUihSQvRT81/SurFHDL7x613ZO/Dts9WeZHxH+++OILoqKi+O6775g4cSLTp09Xd76Ij/jiv+ZRo+Dyy71OIVK8xcfHc91111GzZk1iY2O5+eabyVxBW0T8INeBfMXB7t0Qf/SFBhHJk+3bt1OtWjUiIyOZNWsWzZs3p3Tp0l7HEpFCkNeWfgVjzLqMLyAOMFm3pW+fW3hRc/bSS/D5516cWaR4mzZtGmeeeSbvvPMOAG3atFHBF/GxvLb0DwKP52G/msD9Jx7nxFSpApGRgT6rSPG1b98+hgwZwtSpU7noooto3bq115FEJADyWvQPWWun5LZT+lz8AS/6IpJ3K1asoEePHvzxxx+MGDGCRx55hFKlfHGlT0Ry4Yv/0idNgoRm0Lev10lEir4//viDpKQkFixYQJs2bbyOIyIBlO+ib4wZCFzEsbfxAVQ46UQnYN48qHpARV8kJzt27OD777/niiuu4Prrr+fSSy/VMrgiJVBeir7hyAF/ZYDKpN+rf5SyBREqv266Ca5p6sWZRYq+BQsW0KtXLxITE9mwYQMVK1ZUwRcpofJS9EunfwFgrX0JeCm7HY0x5wABXWEPoGNHuLhxoM8qUrSlpKQwYsQIRo8eTf369fniiy+oWLGi17FExEO5Fn1r7Q9kKfq5CAXCTyqRiJy05ORk2rdvz+LFi7npppt46aWX1LoXkYKZkc8Yc54xJhhYDVQriGPmR0wMrFkT6LOKFF0hISF06dKF6dOnM2nSJBV8EQHyUPSNMS2MMTnul17sVwJVgWCgRsHFy5vnnoNp0wJ9VpGiJSkpidtvv52vv/4agEceeYSePXt6G0pEipS8tPTf4Tjd+9baVNxgv0NAb2Bu+geBgGnaFM45J5BnFClafvrpJ5o3b864ceNYunSp13FEpIjKy0C+w8AhY8yI9O+zW0nP4m7huwv4X/oHgYC59164SgP5pASy1jJhwgTuuusuypUrx6xZs7jkkku8jiUiRVRein5Gkf83sApoDXwPXAD8Rub9+o2AM4D2BZxRRHLw6aefMnjwYDp16sTUqVOpXr2615FEpAjLz0A+C3TGdeVfk/74HDAy/fnVwLvW2r8LOmRu0lLB2kCfVcQ7+/btA+CKK65gxowZzJo1SwVfRHJ1IqP3bfrX0dteB/5z0olOQO8+8PLLXpxZJLBSU1N58sknOeOMM9i4cSNBQUF0796doKACuRFHRHwux+799BH7E3Cz77XBjcz/5+Vs3rLDWru3YOPlTZCBkBAvziwSOFu3bqV3794sWLCAnj17UqGCJ7Nei0gxdrxr+iG4pXLLAl/gJt4pkqa9DVdGeZ1CpPDMnDmT/v37c+DAAd544w369++PMdl99hYRyVmOfYLW2kPW2q7ARlzhT8jlWGcbY64vyHD5ob9/4mczZsygZs2axMbGMmDAABV8ETkheV1lz+bwmFUnoD/w/klmEhHgt99+Iy0tjbPOOotx48ZRqlQpSpfO64zYIiLHyuvoH5P+tTT9cW769keAMenPJwChxpiuBZowD54aDYsWBfqsIoXnrbfeokmTJtx6660AlC1bVgVfRE5aflr6o9KfTz7qNYMbtX8QeB4YBHyZ04GMMZOAc4GZ1tpRx9mvGjDLWnt+buFW/2j4O+A3CooUvH379vF///d/vPXWW7Rp04apU6d6HUlEfCQvRT8UKG2tzfZ2POMuLv4HN7p/KjDcGBNirU3OZt9rgGBrbUtjzBvGmHrW2t9yOO+z5HHFvoeHQqtWedlTpOhav349nTt3Zt26dYwYMYJhw4YRHBzQGa1FxOfyUvRfJXPWveyUxrX2w6y124wx7bMr+OnaAe+lP5+Nm93vmKJvjGkPJALb8pCPRudBZGRe9hQpumrWrMk555zDpEmTaNOmjddxRMSHcr2mb6193lp76DivJwF1gO3p3688zuEigC3pz3eRzTK8xphQ4FHgoZwOYoy5xRgTY4yJyS2/SFG2Y8cOBg8eTEJCAmFhYXz66acq+CJSaApkGi9r7QZr8zQR7n4yu+zL5nD+h4DXrLV7jnO+8dbaaGttNMDnn8Nff+U3tYi35s+fT1RUFJMnT+b777/3Oo6IlACBnrszFtelDxAF/JnNPh2B/zPGfA00NsZMzO2gb78NmzcXVESRwpWSksIjjzxCx44dKV++PMuWLdPKeCISEHkdvV9QPgYWGWNqAl2BHsaYUdbaYRk7WGv/6ds0xnxtrR2Y20EvuxRq1CiUvCIF7oEHHuD555/npptu4qWXXiIiIsLrSCJSQpi89coX4AmNqYSbyOcba22eBuodT1iNevb9WQu5MqrmyYcTKUSHDx8mNDSULVu2sHjxYrp37+51JBEppowxsRmXuPMj4EtzWWt3W2vfK4iCL1IcHDhwgMGDB3P55ZeTlpZGrVq1VPBFxBO+WI9zy2ZIzukmQREP/fjjjzRv3pzx48fTpEkT0tLSvI4kIiWYL4r+fffBNvUbSBFireX111+nWbNm7Ny5k9mzZzNmzBhKlQr0MBoRkUy+KPo1a0JYmNcpRDLt37+f0aNH07ZtW+Li4ujUqZPXkUREAj56v1A895xm5JOiITY2lkaNGlGuXDm+/fZbatWqRVCQLz5bi4gP6K+RSAFITU3lySefpEWLFowdOxaA0047TQVfRIoUX7T0Rby0detWevfuzYIFC+jZsyd33HGH15FERLLli2bInXfCgQNep5CSKGMq3aVLl/LGG2/w9ttvU758ea9jiYhkyxct/fh4UC+qeKFq1aqceeaZvPnmm5x99tlexxEROS5flMoXXtDofQmcX3/9laeeegqARo0asWTJEhV8ESkWfFH0q1cHY7xOISXB1KlTadKkCc8++yxbt24FwOiXT0SKCV8Uff3NlcK2b98++vTpQ79+/WjatClxcXHUrKn1HkSkePFF0Z861esE4mfWWtq3b8/06dMZMWIE8+fP59RTT/U6lohIvvliIN/s2V4nED9KS0vDGIMxhkcffZSKFSvSpk2b3N8oIlJE+aKl37eP1wnEb+Lj47n88st55ZVXALjyyitV8EWk2PNF0e98idcJxE/mzZtHVFQU8+fPJzQ01Os4IiIFxhdFX6QgJCcn8/DDD9OpUycqVarEsmXLGDx4sNexREQKjC+K/s8/eZ1A/CAmJoYxY8Zw8803s3z5cs477zyvI4mIFChjrfU6w0kJq1HPVj59IX8t1e1TcmLWrFnDOeecA8Dq1atp1KiRx4lERI7PGBNrrY3O7/t80dIvE+51AimODhw4wODBg2nYsCFLly4FUMEXEV/zxS17L7zodQIpbn788Ud69OjBTz/9xIMPPkiTJk28jiQiUuh8UfSDNCWf5MPEiRO54447qFChArNnz6ZTp05eRxIRCQhfdO+L5EdCQgJt27YlLi5OBV9EShRfDOSr0/AbfplTw+soUoR9++237Nu3jy5dupCWlgZAkNZjFpFiqkQP5Ev/Gy5yjNTUVEaNGkXbtm159NFHsdYSFBSkgi8iJZIv/vI9+6zXCaQo2rJlCx07duTRRx/lhhtuYN68eVoGV0RKNF8M5AsO9jqBFDVbtmwhKiqKpKQk3nzzTfr166eCLyIlni+KvkgGay3GGGrWrMmQIUPo0aMHZ599ttexRESKBF9077/9ttcJpCj49ddfadOmDWvWrMEYw4gRI1TwRUSy8EXR/+UXrxOIl6y1TJkyhSZNmvDzzz+zdetWryOJiBRJvij6vXp5nUC8sm/fPvr06UP//v2Jjo5m1apVdOjQwetYIiJFki+KfvpaKVICPf/887zzzjuMHDmSefPmUatWLa8jiYgUWRrIJ8VOWloa27Zto2bNmjz44IN06dKF5s2bex1LRKTI80VLPzbW6wQSKPHx8Vx++eVceOGF7N+/n7CwMBV8EZE88kXRnznT6wQSCPPmzSMqKor58+dz//33ExER4XUkEZFixRdFX6ui+ltKSgoPP/wwnTp1olKlSixbtozbb79dk+2IiOSTL4r+5Zd7nUAKkzGGJUuWMHDgQJYvX855553ndSQRkWJJA/mkyPrwww9p1aoV1atXZ9asWZQuXdrrSCIixZovWvr793udQArSgQMHuOWWW7j22msZO3YsgAq+iEgB8EXRf/VVrxNIQVm9ejXNmjVj4sSJPPTQQ4wZM8brSCIivuGL7v2yZb1OIAVh1qxZdOvWjQoVKvDVV1/RqVMnryOJiPiKL1r6//d/XieQgtC8eXN69OhBXFycCr6ISCHwRdGX4mvx4sVcd911HD58mMqVK/Pmm29SrVo1r2OJiPiSir54IjU1lSeeeIK2bdvyww8/sGXLFq8jiYj4ni+K/vTpXieQ/NiyZQsdO3bkscceo0ePHqxYsYI6dep4HUtExPd8MZBv506vE0h+9OzZkxUrVjB58mT69u2rmfVERALEF0W/Rw+vE0huDh06RGpqKmXKlOH1118nODiYs846y+tYIiIlii+69yMjvU4gx7N27VouuOAC7rjjDgDOPfdcFXwREQ/4ouhL0WStZfLkyTRt2pRNmzbRrVs3ryOJiJRovij6y5d7nUCOtnfvXnr37s2AAQNo1qwZcXFxXK6VkUREPOWLoh8b63UCOdrff//NrFmzeOKJJ5g7dy61atXyOpKISInni4F80dFeJxCAtLQ0Pv74Y7p160adOnX4448/qFixotexREQknS9a+ir63ouPj+eyyy7j2muvZebMmQAq+CIiRYwvWvrirblz59KnTx92797Na6+9xmWXXeZ1JBERyYYvWvqanMc7zzzzDJ07d6ZSpUosX76c2267TZPtiIgUUb4o+p995nWCkuv8889n4MCBxMTE0KhRI6/jiIjIcfiie/+UU7xOULK89957bNiwgfvvv59OnTppGVwRkWLCFy39K67wOkHJkJiYyKBBg+jevTuffPIJKSkpXkcSEZF8CHjRN8ZMMsZ8Z4wZlsPrFYwxXxpjZhtjPjLGhAY6oxxr1apVREdHM2nSJIYOHcqCBQsoVcoXHUUiIiVGQIu+MeYaINha2xKoa4ypl81uvYDnrLWdgW1Al0BmlGPt3r2b1q1bs2fPHubMmcPo0aMJCQnxOpaIiORToFv67YD30p/PBlofvYO19jVr7Zz0b6sC8UfvY4y5xRgTY4yJAXj//cIJW9IlJSUBUKlSJaZMmUJcXBwdOnTwOJWIiJyoQBf9CGBL+vNdQLWcdjTGtAQqWWu/P/o1a+14a220tTYa4PDhwohasi1evJizzjqLTz75BIBu3boRqeUMRUSKtUAX/f1AePrzsjmd3xhTGXgZuCkvB7322gLJJkBqaiojR46kbdu2hIaGas58EREfCXTRjyWzSz8K+PPoHdIH7r0PDLXWbsjLQUuXLqh4JdvmzZvp0KEDw4cPp2fPnqxYsYJozXEsIuIbgS76HwN9jDHPATcAPxljRh21z81AE+ARY8zXxpjuAc5YYs2fP5+YmBimTJnCtGnTKF++vNeRRESkABlrbWBPaEwloBPwjbV228keL6xGPfvQE9/w+MAaJx+uBDp06BArVqygZcuWWGv566+/qFmzptexRETkOIwxsRnj2vIj4PfpW2t3W2vfK4iCn2H9+oI6Usmydu1aLrjgAjp16sSOHTswxqjgi4j4mC9m5Gve3OsExYu1lsmTJ9O0aVM2bdrEjBkzqFq1qtexRESkkPmi6Net63WC4iM1NZU+ffowYMAAmjVrRlxcHJdffrnXsUREJAA0j2oJExwcTGRkJE888QRDhw4lODjY60giRc7evXuJj48nRyPHugAAGvdJREFUOTnZ6yhSwoSEhBAZGVloA6l9UfS3bAG0qmuO0tLSeO6557joooto0aIFzz33nNeRRIqsvXv3sn37dmrVqkV4eDjGGK8jSQlhrSUpKYktW9wcdoVR+H3Rvb9kidcJiq7t27dz6aWXcv/99/POO+94HUekyIuPj6dWrVqUKVNGBV8CyhhDmTJlqFWrFvHxx8xAXyB80dLXgPPszZ49m759+5KQkMC4ceMYPHiw15FEirzk5GTCw8Nz31GkkISHhxfapSVfFP0LL/Q6QdEzb948LrnkEs4991zmzp1Lw4YNvY4kUmyohS9eKszfP19070um1NRUANq1a8ezzz7L8uXLVfBFRATwSdFPS/M6QdHw7rvvcu6557Jt2zaCg4O59957KVOmjNexRMRjO3fu5MYbb6RSpUpERkby6KOP/vPawYMHufXWW6lQoQLVqlVj9OjR/7w2YsQIjDEEBQURGRnJDTfcwNq1a734EaSA+KLof/yx1wm8lZiYyMCBA+nRoweVK1fWbUYicoTu3buzdetWPvjgA4YOHcpTTz3Fu+++C8Cdd97JzJkzmTZtGiNHjuTxxx/ngw8++Oe9NWrU4Pvvv+eFF15g1apVtGrVio0bN3r1o8hJ8sU1/ZJ8+W3VqlV0796dtWvX8vDDDzNixAhCQkK8jiUiRcSff/7J/PnzWbFiBeeffz7t27dn0aJFTJ06lTZt2vDGG28wbdo0rrjiCgC+++47Xn75Za5NX7M8NDSU5s2b07x5c9q3b0/9+vV56qmnGDdunJc/lpwgXxT9bt28TuCd0aNHk5CQwJw5c+jQoYPXcUSkiNm1axfguvgzPPPMMyQkJDBv3jxSU1Pp1KnTP6+df/75fPHFF9keq3r16lxxxRU5vi5Fny+690taQ3/Xrl1s2rQJgNdee424uDgVfBHJVoMGDTjttNPo378/H374IdZazjzzTJo2bcovv/xCuXLlqFKlyj/79+vXjwULFuR4vPPOO4+NGzeSlJQUiPhSwHxR9EuSRYsW0bhxY2688UastVSuXFmL5YhIjsLCwvjss88ICwvj2muvJTo6mu+++w5wrf+jZ32rWLEiDRo0yPF4lSpVAmDPnj2FF1oKjS+K/pLvvE5Q+FJTUxk5ciTt2rUjLCyMF154QfcSiwSYMceOIbriCrfts88yt40f77bdckvmtq1b3bajJxNr2tRtj43N3DZihNs2YkTmtqyv51dUVBS//PILr732Glu3bqVdu3bMnDmT5ORkgoJcGfj+++8xxvzzlRP93SnefFH0dxTObIVFRnx8PB06dGD48OHceOONrFixgqZNm3odS0SKkdDQUG677TZWr17NOeecw+DBg4mIiCAxMRH4//buPTqKKk/g+PdHDIQENsQgATwj0cAqImuMysMn0WHFqAziI0yCiDyHURQdByHyiuIKq+bs6KCMKKA4Or4YBRkwgCbxARIwoigeg7xcUMCIDxIwG/LbP6rTdEJIOiHdTVd+n3P62F1169avr01+Vbeq7nW67YuKipg3b16d9ezfvx+A2NjYgMdsmp4rkn6fvqGOILBiYmIoKyvjueeeY9GiRbRt2zbUIRnTLKk6L19LlzrLPDe/A84Zvqpzxl+lc2dn2e7d1bffsMFZ7nscP2OGs8z3TL+xx/nz5s1jwIAB3s/t27dn6tSp7Nq1i/j4eH744Qd++uknoqOjSU5OJiEhoc76Nm3aRGJioo0BEqZckfQTOoQ6gqZ36NAhZs6cSWlpKTExMaxdu5Zhw4aFOixjTJiJiopi9erV1a7Bl5SU0Lp1awYPHgzAUp9rExs3bjxmXfv27WPJkiUMGjQocAGbgHLFI3tu8+WXXzJkyBA2btxIt27dSE9P9153M8aYhrjuuuuIi4vjxhtvZNKkSezdu5fp06czZswYevbsyU033cQdd9wBQERExFFTb5eXl1NYWMjXX3/NzJkzadu2LVlZWaH4KqYJuCKTbNse6giahqoyf/58zj//fHbt2sVbb71Fenp6qMMyxoSxdu3asWrVKiorKxk8eDCTJ09m2LBhzJ49G4CFCxdy0003MW7cOGbMmMHtt99ebftvv/2W3r17M2HCBHr16sVHH31kTwyFMdGaF6jCTKtO3fT6oQX845FOoQ7luD344INMmzaN1NRUXnjhBTrbnMHGBN3mzZvp3r17qMMwzVx9v0MR2aCqFzS0Xld071/Q4K99YlFVRITMzExatmzJvffeS0RERKjDMsYY4zKu6N7vcXaoI2icyspKHnnkEW6++WZUlTPOOIP77rvPEr4xxpiAcEXSD0d79uwhLS2NiRMnUllZyaFDh0IdkjHGGJdzRdIvC7MhoHNzczn33HPJz89n7ty5vPbaa7Ru3TrUYRljjHE5VyT9goJQR+C/srIyhg8fTnx8PIWFhYwdO9aGtTTGGBMUrriRLxxOkr/55hs6d+5MdHQ0b7/9NklJSTailTHGmKByxZn+5ZeFOoK6vfzyy5xzzjne52J79uxpCd8YY0zQuSLpn6hKS0sZNWoUQ4YMoUePHmRkZIQ6JGOMMc2YJf0A+eyzz7jggguYP38+WVlZ5Ofnk5iYGOqwjDHGNGOuSPofrgl1BEc7ePAgZWVlrFq1ioceeojIyMhQh2SMaWYWLlyIiCAitGjRgi5dunDvvfd6p9MN1D6DdYKTl5fn/X41X3l5eUGJIdy44ka+0gOhjsBRUlLCm2++yYgRI+jVqxfFxcW0bNky1GEZY5q5wsJCysvLWbduHVOnTmXPnj0sWrQo1GE1mSVLltCpU/Wh2M8888wQRXO0vLw8tm/fzvDhw0MdijuSfp8+oY4ACgoKyMzMZO/evVxxxRUkJiZawjfGnBAu8IxVftFFF1FaWsoDDzzAM888Q6tWrUIcWdPo2bPnCX35NC8vj7y8vBMi6buie79t29Dtu6KighkzZpCamkrr1q1Zs2bNCf3jM8Y0bykpKZSXl1NSUhLqUEwIuCLph4qqMnDgQLKzs8nMzGTDhg2kpKSEOixjjDmmPXv2ICLEx8cDsGvXLgYNGkRsbCwdO3bk7rvvprKyEoDt27cjInzyySfceOONtGnThrPOOos1a47cSPXFF19w8cUXExUVRd++fdm2bVu1/e3fv5+hQ4fSpk0bOnbsSHZ2NlWzu/pT//Gqa/9w5B6EgwcPcvfdd9O5c2fy8/O963/55RdGjx5NXFwcHTt25J577qG8vNy7vqr94uLiaN++PSNGjPAOqz58+HBEhOzsbPLz8733GyxcuLDJvl9DuSLpb/k6NPsVETIyMnj++ed5/vnnaRvKLgdjjKnH559/zqxZs7jyyiu9XfuZmZl8/fXXLF68mDlz5nj/nvm65ZZb6NSpE2+88QYxMTH88Y9/BJyezkGDBqGqvPnmm6SlpfHwww9X23bIkCEUFhby4osvkp2dzaOPPsqsWbP8qr8p+LN/gMGDB7Np0yYmT55M165dvctHjx5NQUEBixYt4oknnmDRokVMmzbNu37cuHFs2bKFV155hblz55Kbm8tjjz0GwIwZMygsLGT06NGkpKRQWFhIYWEh1113XZN9v4ZyxTX9HduDt69Dhw7x5z//mZSUFG677TaGDh0avJ0bY0ImcdKyUIcAwPZZ1zR4G9+hvlNSUnj22WcBp7cyIyODiy++mB49elBRUcGcOXP46KOPql1/7t69O0888QQAWVlZDBkyBHDmESkuLmb58uUkJSVx1VVXUVRUxMcffwzA+++/T25uLkVFRSQnJwPOUORTp07lnnvuqbd+f51++unVPledyde3/6oDnx07dtCnTx9eeumlam21detWXn75ZfLy8rj88ssBp2fjmWee8R447Ny5k8suu4z+/fsDkJSU5K0jMTGRxMRE3nrrLb766ivvvRWh5Iqkn9S1/jJNYfPmzQwZMoRPP/2UrKys4OzUGGOOU1FREbt37+baa69l4sSJnHbaaYBzMHDzzTczf/58Jk6cyNq1a9m/f793fZUxY8Z438fHx1NRUQFAcXExJ598MklJSd71l112mTfpf/LJJ8TGxnoTLkC/fv0oLS1ly5YtxMTE1Fm/v5YtW0bnzp2PWl7f/nv06AFAy5Yteeyxx46aB+Xzzz/3blNTeXk5LVu2ZOzYsYwfP57i4mL69OnDtddeS+/evRsUfzC5Iukndgls/arKggULGD9+PNHR0Sxbtoy0tLTA7tQYc0JpzBn2iSI5OZnk5GQGDhzI7NmzSU9PB5zr1SkpKXTo0IGMjAymTp3KU089ddT2Nc+kq1RWVtKiRfWrxBEREdU+10ykVZ99r6sfq35/nX322ce8gdqf/Xfq1IlTTz31qG2rynzwwQdHDZ1+0klO+hw3bhyXX345ubm5vPPOOzz00EPk5OQwYcKERn+fQHLFNf1AW79+PSNHjqR3795s3LjREr4xJixlZWVRVFTEypUrAVi9ejXbtm1j+fLl3HnnnfTp04ctW7YctV3NRF4lKSmJkpISdu7c6V32wQcfeN8nJyfz448/8umnn3qX5efnEx0dTbdu3eqt/3j5u/9jqeoJqKys9B44HTx4kJycHG9vxKRJk1BVJkyYwJIlSxgzZgzz58+vVk9UVBQHD54Yc8C7Iun/9FNg6t23bx8AF154IStWrGDlypW1diEZY0w46NWrF1deeaV38q+qO/gXLFjAypUrGTRoEB9++KHf3esDBgygS5cu3HLLLaxcuZJZs2bx+uuve9dfcskl9O/fn/T0dJYuXcq8efOYNm0aU6ZMCcoYAce7/6SkJNLT0xk5ciSLFy9m+fLljBo1ipKSEu84LOvWreOuu+5i1apVrFixgry8vKN6Li688EKKiopYvHgxBQUFPP744wH5vn5R1bB+tezYVSc+vlub0uHDh3X27NkaHR2t69ata9K6jTEnti+++CLUITSZBQsWqPNn/oh33nlHAS0sLFRV1fvvv1/j4+M1ISFBhw8frmPHjtWuXbtqRUWFbtu2TQHdtm2bd/t33323Wp2bN2/Wfv36aXR0tKakpOh9992nXbp08a7/4YcfNCMjQ2NiYrRDhw46ffp0PXz4sKqqX/XXpaqs7/Y11bX/qjbyjbemn3/+WUeOHKnt2rXTuLg4HTZsmJaUlHjX79ixQ6+//no9+eSTtU2bNnrNNdfozp07j6onOztbO3TooJGRkZqWllbvd6vvdwis10bkTFGf6xrhqFWnbjptdgH3D+tUf2E/7Nmzh2HDhpGbm8sNN9zAvHnziIuLa5K6jTEnvs2bN9O9e/dQh2Gaufp+hyKyQVUb/DiAK7r3U85rmnpyc3M599xzKSgoYO7cubz66quW8I0xxriGK+7ebypr1qyhffv2rF692nsDhzHGGOMWrjjTPx5bt2713m06ZcoUCgsLLeEbY4xxJVck/Q8bOUzzSy+9RHJyMqNGjeLw4cNERETQunXrpg3OGGOMOUG4Iuk3cPAmSktLGTFiBBkZGfTs2ZMVK1YE7DlRY0z4CfcbnE14C+TvzxXX9Pv29b/svn37uPTSS/nqq6+YMmUK06dP946sZIwxJ510EhUVFURGRoY6FNNMVVRUBCwvuSLbRTbgW7Rv357U1FSeeuopUlNTAxeUMSYsRUVFceDAAXtyx4TML7/8QlRUVEDqdkX3fn1KSkrIzMxk69atiIglfGPMMZ1yyins27ePsrIy6+Y3QaWqlJWV8f3333PKKacEZB+uONMvLoare9a+Lj8/n8zMTPbu3cvAgQM544wzghucMSasREVFkZCQwHfffcevv/4a6nBMM9OqVSsSEhICdqbviqS/d9/RyyoqKpg5cyYPPvggSUlJrF27lpSUlOAHZ4wJO7GxscTGxoY6DGOanCu697t2PXpZTk4O2dnZDB06lA0bNljCN8YY0+y54kw/ocOR9wcOHKBNmzbcfvvtJCUlccMNN4QuMGOMMeYE4oozfYBDhw5xxx130KtXL0pLS4mJibGEb4wxxvgIetIXkWdFZI2ITDmeMr4+/biY3r17M2fOHAYMGGDP3RtjjDG1CGrSF5HBQISq9gXOEJFujSnj63DZz0wdM4Ddu3ezbNkycnJyaNWqVWC+gDHGGBPGgn2m3w94xfM+F7ikkWW8Kg/+ROK/n8/GjRtJS0trojCNMcYY9wl2P3gMsMvz/gegtlvq6y0jImOAMZ6PvxZven/Tqaee2sShmhraA9+HOgiXszYOPGvjwLM2Do4zG7NRsJP+AaBqGrs21N7TUG8ZVX0aeBpARNar6gVNH6rxZe0ceNbGgWdtHHjWxsEhIusbs12wu/c3cKS7/lxgeyPLGGOMMaaBgn2m/wbwnoh0Bq4GhojITFWdUkeZPkGO0RhjjHGloJ7pq+rPODfqrQVSVXVjjYRfW5mf6qn26QCEao5m7Rx41saBZ20ceNbGwdGodhabRcoYY4xpHlwzIp8xxhhj6hY2ST8QI/mZ6uprPxGJFZHlIpIrIv8UkZbBjtEN/P2dikiCiBQFKy43aUAbPyki1wUrLjfx4+9FnIj8S0TWi8jfgh2fW3j+DrxXx/pIEVkqIh+IyIj66guLpB+IkfxMdX62XyaQo6r/CXwHDAhmjG7QwN/poxx5fNX4yd82FpFLgY6qujSoAbqAn218C/B3z+N7bUXEHuNrIBGJA57DGb/mWMYDG1T1YuBGEWlbV51hkfQJwEh+5ij9qKf9VPVJVV3p+XgKsDc4oblKP/z4nYrIFUApzsGVaZh+1NPGIhIJzAO2i8jvgheaa/Sj/t9xCXCOiLQDfgN8E5zQXOUwkA78XEeZfhz5f1EA1HlwFS5Jv+YofQmNLGOOze/2E5G+QJyqrg1GYC5Tbzt7LptMBSYFMS438ee3PAz4AvhvoJeIjA9SbG7hTxu/D3QB7gQ2e8qZBlDVn/14gq1BuS9ckn6TjORn6uRX+4nIycATQL3Xjkyt/GnnScCTqvpj0KJyF3/a+DzgaVX9DngBSA1SbG7hTxtPB/6gqg8AXwK3BSm25qZBuS9cEqON5Bd49baf5wz0VWCyqu4IXmiu4s/v9LfA7SKSBySLyDPBCc01/GnjLcAZnvcXAPZ7bhh/2jgO6CkiEUBvwJ4PD4wG5b6weE5fRP4NeA9YjWckP+Am34F9ainTx49uEePhZxuPA/4L2OhZ9JSqvhzsWMOZP+1co3yeqvYLXoThz8/fcltgPk5XaCRwo6ruqqU6Uws/27gXsACni38NcL2qHghBuGGv6u+A516fs1X1rz7rugD/AlYBF+HkvsPHrCsckj5472LsDxR4uuQaVcYcm7VfcFg7B561ceBZG584PMPWXwK8Xd/JbtgkfWOMMcYcn3C5pm+MMcaY42RJ3xhjjGkmLOkbYwJORCJEREIdhzHNnSV9Y1xCRAaJyEXHWBcVyLkSROQqEfmTz+eHReRtnyLTgKWex7f8qS/TMwiUMaYJnRTqAIwxTWYq8I6I5OA8F11lMpAI/F5EFGcAj9tV9W8AInIecIj6n6OOAKKAz1S1vMa6n4AsEemgqvcBvwIHPfWnAROBjJqPEnmGwz0J+FVVK31WjQDKgOt8ykYALYFKVf21nliNMbWwpG+MC4jIaUBPYCDQC0hV1TwRWYiTUP8A/MFTNg9nFK8qa3GStG/SbYmT4H3H/G7hWX4mnsFsRKQVIMA64BrgL7VM+HEPME5Vq2ZmbKGqhzzr0oG/AgdFpCqRR+IcgFSIyHafeiKBaJyJiB7yq2GMMdVY0jfGHW7FmWlrl+dsvj7eM25VbVVzpYgMB2aoamI99cwG7qqxzHug4BPLlSKywPP+TWCQ5/1Lnv+uUNXvPdu8BLTzlEkGClW10jOT2zU4w0AbYxrBrukbE+ZE5CRgFM7ZepV3PQn3ViBKRFaJyC8i8iPOIB51TdXZEDOAeCBSVQXoCnwLfIWT3LsDS3AuL7TA6T34vc/2rYE+wFci0l9EXgM64ozbngXkA1d75glfD3Ty1GGMaQQ70zcm/N2Gc53eV6qq5lV9EJG/4HTf/6q1jMglIncBZao6ryE79p0UyDNE6Iue1y84k9ocwDnA2AT8SVWfrrH9AeAOEXkaKAduwOkpWIlzQHC9qi7z3Hdwq6q+0ZD4jDHV2Zm+MWHMcy1/FvDkMdZHiUgHnKlkfw/cKiLDReScGkX7A5fVWNZCRNr5vNqLSKda9nGup0t+KfCwqt6Dc+2/lar+r6fu6cAcEVnqiaemUuBHnPsSrsC5T2AocKqIpODMzd7VHvsz5vhY0jcmvO3GSagbaiyv6t4/CPwGeATIxLlOngN0q1G+Ap/r/B6/Afb7vPYBy30LiMiFwMc4N9glq+pffOL6UUQi1JGDMzHLadT4uyMiVwMfAVfi9Aq8iXP3/uvAVTgT4yTiPIXwmohE19coxpjaWdI3JoypaoXvjFs+Uj3X2FvjJOV/AX9V1UE43ejr/ah+h6pK1Qvn7vma4wBsAnqo6u9UtbjGul74PBGgqqs8y7xPDohIFvA8Tk/EZpzZ2NoCj+M8Btgb6AGk4EyB+x84d/wbYxrBrukb41KervCq7vBVwAAR+RTn2v03Da1PVStwegR8LQcur6PXvfIY60REWgD/AF5R1S2ehb1x5gN/FtiqqhM8U7R+o6rfikgyoJ4ehGNOH2qMqZ0lfWPc6V2f96cDi3FGxVNgThPuJ81Tp3dwHRE5E/gE2AUsVdW7qwp7ntOvekSwJ85liXIRqTnYTwzOAcNwn23hyFgBV+Hc2W+MaQBL+sa4U9XgPJFAhaqqiOTiXCvv2FQ7UdUy38+eeb3/jtNlnw186OlxmKyqBz0j+ZV7tt3IMf4GicgbwHZVndBUsRpj7Jq+MW4RwZF/z5FVC1X1/4A2IjIVGABsBOaLSIJnEpxkEemO88hfrIicJSJn4TwPH1n12fM621O+a82di0i8iNyNc4a/GbhTVXcDfXGuxW8SkfEiEhu4JjDG1MfO9I1xhyiODFrjHWHPMwHPCpw74lNw7sL/H+ALnJviPqL6uPtra9Rb83Okp74bPPVPwHkU8Dzgc2Csqv6zqrDnOnw/YAzOQD45IvKyqg6t5/v4HsQYY5qI1DJOhzHGRUQkQVX31FjWvmrY2+Os+2Lgt8Abnu76usq2wnlkcLeqvldP2ZXANlUdc7wxGmOOsKRvjDHGNBPWfWaMMcY0E5b0jTHGmGbCkr4xxhjTTFjSN8YYY5oJS/rGGGNMM2FJ3xhjjGkm/h/YDU7Ux5PQFAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,6))\n",
    "plt.plot(fpr,tpr,\"b:\",linewidth=2,label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest,tpr_forest,\"Randon Forest\")\n",
    "#plt.plot(fpr_forest,tpr_forest,\"b--\",linewidth=2,label=\"Randon Forest\")\n",
    "plt.legend(loc=\"lower right\",fontsize=16)\n",
    "plt.show()       \n",
    "\n",
    "# 实线为随机森林分类器，虚线为SGD分类器曲线，由下图可知，随机森林分类器的效果优于SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9915758753579609"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rand 比 SGD 好很多，ROC AUC的分数也高很多\n",
    "roc_auc_score(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9828533743679929"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 再看一下，随机森林分类器下的精度和召回率  也很高\n",
    "y_train_pred_forest = cross_val_predict(forest_clf,X_train,y_train_5,cv=3)\n",
    "\n",
    "precision_score(y_train_5,y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8247555801512636"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5,y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结（重要思路）\n",
    "   * 在未确定使用哪个模型之前，可使用ROC曲线和ROC AUC 分数来比较多个模型\n",
    "   * 获取到较好的模型之后，使用PR曲线调节阀值，选择满足条件的精度/召回率权衡\n",
    "   * 关于决策树的参数，可使用网格搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多类别分类器\n",
    "   * 尝试5之外的检测\n",
    "   * 多类别分类器 区分两个以上的类别\n",
    "   * 随机森林和朴素贝叶斯可以直接处理多个类别\n",
    "   * 支持向量机svm和线性分类器只可以处理二元分类器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 解决方案\n",
    "   * 将数字图片分类0到9，训练10个二元分类器，检测一张图片时，获取每个分类器的决策分数，哪个最高属于哪个，称为一对多OvA\n",
    "   * 为每一对数字训练一个二元分类器，区分0,1  区分1,2等等，称为一对一OvO策略，存在N个类别，需要N*(N-1)/2 个分类器，最后哪个类别获胜最多"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 优缺点\n",
    "   * OvO只需要用到部分训练集对其必须区分两个类别进行训练\n",
    "   * 对于较小训练集合，OvO比较有优势，大训练集合OvA速度快，所以OvA更常用，，比如svm在数据规模扩大时表现糟糕\n",
    "   * sklearn 检查到使用二元分类算法进行多类别分类任务，会自动运行OvA，svm分类器除外"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用SGD线性二元分类器进行多类别分类\n",
    "\n",
    "sgd_clf.fit(X_train,y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -4265.99131721, -21876.03114374,  -9830.28939646,\n",
       "         -2853.87674806, -15834.621569  ,   3086.69640423,\n",
       "        -21161.21605523, -13145.94199438,  -6934.09202832,\n",
       "        -18447.74172658]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 内部实际上训练了10个二元分类器，获得图片的决策分数，然后选择了分数最高的类别\n",
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)         # 选出决策值做大的索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_                     # 返回所有分类的分类标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_[np.argmax(some_digit_scores)]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 当需要强制执行OvO策略，一对多或者一对一\n",
    "\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(max_iter=5,tol=-np.infty,random_state=42))\n",
    "ovo_clf.fit(X_train,y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)          # 查看分类器的个数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用随机森林 使用多元分类器来预测值\n",
    "\n",
    "forest_clf.fit(X_train,y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.1, 0. , 0. , 0. , 0. , 0.9, 0. , 0. , 0. , 0. ]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机森林直接将实例分为多个类别，调用predict_proba()可以获得分类器将每个实例分类为每个类别的概率列表\n",
    "\n",
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.87232553, 0.87269363, 0.8840326 ])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用交叉验证评估SGD分类器的准确率\n",
    "cross_val_score(sgd_clf,X_train,y_train,cv=3,scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:561: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.90406919, 0.8980949 , 0.89728459])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将输入进行简单缩放，可以得到准确率 90% 以上\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf,X_train_scaled,y_train,cv=3,scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 错误分析\n",
    "### 项目流程\n",
    "    1.探索数据准备的选项\n",
    "    2.尝试多个模型\n",
    "    3.选择最佳模型并用GridSearchCV对参数进行微调\n",
    "    4.尽可能自动化（转换流水线）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 确定了一个相对合适的模型，进一步优化，分析其错误类型\n",
    "   * 查看混淆矩阵\n",
    "   * 使用cross_val_predict()进行预测\n",
    "   * 调用confusion_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:561: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[5598,    0,   14,    9,    9,   35,   36,    5,  216,    1],\n",
       "       [   1, 6417,   43,   21,    3,   43,    4,    9,  191,   10],\n",
       "       [  26,   27, 5245,   85,   68,   26,   78,   42,  352,    9],\n",
       "       [  26,   22,  106, 5245,    1,  207,   25,   41,  391,   67],\n",
       "       [  12,   14,   45,    8, 5222,   10,   33,   27,  316,  155],\n",
       "       [  28,   17,   25,  165,   54, 4460,   77,   13,  518,   64],\n",
       "       [  26,   20,   52,    3,   41,   84, 5543,    8,  141,    0],\n",
       "       [  23,   16,   50,   23,   45,   10,    2, 5726,  165,  205],\n",
       "       [  18,   61,   40,   96,    3,  119,   31,    8, 5428,   47],\n",
       "       [  26,   24,   27,   57,  128,   42,    1,  179,  360, 5105]],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf,X_train_scaled,y_train,cv=3)\n",
    "conf_mx = confusion_matrix(y_train,y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAEACAYAAABxpdD1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAKgElEQVR4nO3dTYid9RWA8eckmUSxWhO1SomIShdSxShDrFBLFCt2IdLUYkEM2JZgEV1HWpG6aEHEjRBx/OhG++UiRcGCpRCqoEjEFEFTcWEQIdgm1aqQz3u6yIgSJ7nvJO//vjMnz281yVz/c7jxyXvv5N4zkZlIqmvJ0ANIasvIpeKMXCrOyKXijFwqzsil4oy8g4j4ekT8NSJejIgtEbF86Jm6iIhzI+KNoeeYj4jYHBE3DT1HFxGxMiJeiIhtEfHY0PMczSCRR8STEfFKRPxqiK9/HG4DHs7MG4BdwI0Dz9PVQ8CpQw/RVURcA5yXmc8PPUtHtwPPZOY0cHpETA890FwmHnlErAeWZubVwEUR8a1JzzBfmbk5M/82+8tzgA+HnKeLiLgO+IzDfykteBExBTwOvBcRNw89T0e7gUsj4kzgfOD9geeZ0xBX8nXAn2c/fhH47gAzHJeIuBpYmZmvDj3Lscw+nbgP2DT0LPOwAXgLeBBYGxF3DzxPFy8DFwD3AG8De4YdZ25DRH4a8MHsx3uAcweYYd4iYhXwCPDToWfpYBOwOTM/GnqQebgCmMnMXcDTwLUDz9PF/cCdmfkAsAO4Y+B55jRE5J/yxfPErw00w7zMXhmfBe7NzJ1Dz9PB9cBdEbEVWBMRTww8TxfvAhfNfjwNLIb7eSVwWUQsBa4CFuQbQWLSb1CJiA3ANzLzoYj4NfCvzPz9RIeYp4j4BfAb4J+zv/VoZv5pwJE6i4itmblu6DnGiYjTgac4/MhuCrglMz849n81rIhYC/yOww/ZXwF+mJmfDjvVVw0R+RnAS8DfgR8A38nMjyc6hHQSmXjkcPjfF4HvA/+YfQ4mqZFBIpc0OQv+m16SToyRS8UNFnlEbBzqax8vZ25vsc0LC3/mIa/kC/qOOQpnbm+xzQsLfGYfrkvF9frd9VWrVuXq1as73Xb37t2cddZZnW775ptvnshY0rxFROfbZua8b99CZs45xLI+v8jq1at57rnn+jwSgAsvvLD3M/VV8/kfdaFoFcyKFSuanAuwd+/eZmfPxYfrUnFGLhVn5FJxRi4VZ+RScUYuFdcp8kW4XVXSrLGRL8btqpK+0OVKvo5Ful1VUrfIj7ldNSI2zv4EiW27d+/uez5JJ6hL5MfcrpqZM5k5nZnTXV+LLmlyukT+Ol88RL8ceK/ZNJJ61+UNKn8BXoqIbzK7XbXtSJL6NPZKnpn/4/A3314FrnV9srS4dHqraWb+ly++wy5pEfEVb1JxRi4VZ+RScUYuFdfrIseIaLJwq+WPclqypM3fc4vxx0+12vG2GO+Lljve9u3b1+Tcoy1y9EouFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxnX4W2ny0WOvbam0ywPbt25uce+WVVzY5F9qtOG517tKlS5ucCzAajZqcuxhXMh+NV3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXihv7YpiI+DrwR2Ap8Blwa2bubz2YpH50uZLfBjycmTcAu4Ab244kqU9jr+SZuflLvzwH+LDdOJL61vm16xFxNbAyM1894vc3Ahv7HkxSPzpFHhGrgEeAHx35ucycAWZmb9fmHQ6SjtvY5+QRsRx4Frg3M3e2H0lSn7p84+1nwJXALyNia0Tc2ngmST3q8o23R4FHJzCLpAZ8MYxUnJFLxRm5VJyRS8UZuVRc9LmhMyKyxbbWVltEAZYt631hLQCvv/56k3MBLr/88ibnnnLKKU3O3bt3b5Nzoc12YIAzzjijybkAn3zySe9njkYjMnPOO8MruVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxfW+krm3wyak1Urflmukt2/f3uTcNWvWNDm31X3c0mmnndbs7P379/d+5oEDBxiNRq5klk5GRi4VZ+RScUYuFWfkUnFGLhVn5FJxnSKPiHMj4o3Ww0jqX9cr+UPAqS0HkdTG2Mgj4jrgM2BX+3Ek9e2YkUfEcuA+YNNkxpHUt2VjPr8J2JyZHx3t9ccRsRHY2Pdgkvox7uH69cBdEbEVWBMRTxx5g8ycyczpzJxuMaCkE3PMK3lmfu/zjyNia2b+vP1IkvrU+d/JM3NdwzkkNeKLYaTijFwqzsil4oxcKs7IpeKMXCqu922tLTZzttx82sry5cubnX3w4MEm527ZsqXJuevXr29yLsChQ4eanHv22Wc3ORdgz549vZ85Go3ITLe1SicjI5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpuJN+W+uSJW3+nhuNRk3OBWhxH0O7++Kdd95pci7AxRdf3OTcqampJucCHDhwoMm5bmuVTlJGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhXXOfKI2BwRN7UcRlL/OkUeEdcA52Xm843nkdSzsZFHxBTwOPBeRNzcfiRJfepyJd8AvAU8CKyNiLu//MmI2BgR2yJiW4sBJZ2YLpFfAcxk5i7gaeDaL38yM2cyczozp1sMKOnEdIn8XeCi2Y+ngZ3txpHUt2UdbvMk8FRE/ASYAm5pO5KkPo2NPDM/AX48gVkkNeCLYaTijFwqzsil4oxcKs7IpeKMXCqu95XMvR02Ia3WG7dcI70YZ27l/fffb3Juq1XP0Gb19b59+xiNRq5klk5GRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScYtiW2uL7Zafa7WhtOXMhw4danLusmVdfsjt/LWaF9r9+e3YsaPJuQCXXHJJ72dmJpnptlbpZGTkUnFGLhVn5FJxRi4VZ+RScUYuFXfMyCNiZUS8EBHbIuKxSQ0lqT/jruS3A89k5jRwekRMT2AmST0aF/lu4NKIOBM4H2jzw6AlNTMu8peBC4B7gLeBPc0nktSrcZHfD9yZmQ8AO4A7jrxBRGycfc6+rcWAkk7MuMhXApdFxFLgKuAr7wbIzJnMnJ593i5pgRkX+W+BGeBjYBXwh+YTSerVMd9bmJmvAd+e0CySGvDFMFJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxve/gjZhzK+wJGY1GvZ/5uRbzQrtVwQBTU1NNzj148GCTc1veFytWrGhy7tq1a5ucC/Daa6/1fuaGDRuO+jmv5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScdHnJs2I+Dews+PNzwb+09sXnwxnbm+xzQsLY+YLMvOcuT7Ra+TzERHbMnN6kC9+nJy5vcU2Lyz8mX24LhVn5FJxQ0Y+M+DXPl7O3N5imxcW+MyDPSeXNBk+XJeKM3KpOCOXijNyqTgjl4r7P9L4R26yj9UYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用matplotlib 的matshow函数来查看混淆矩阵的图像表示\n",
    "plt.matshow(conf_mx,cmap = plt.cm.gray)       # 该颜色域表示数值越小颜色越深，数值越大颜色越白\n",
    "plt.show()               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 由图可知，数字5看起来比较暗，有可能图片5较少，可能该分类器在数字5上执行效果不如其他数字好\n",
    "# 假设把焦点放在错误上，为取得错误率，而不是错误绝对值，需要将混淆矩阵中每个值除以相应类别中的图片数量\n",
    "\n",
    "row_sums = conf_mx.sum(axis = 1,keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAEACAYAAABxpdD1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAALKklEQVR4nO3dX4yeZZmA8euer6WdlKJDqVgTUtJke8Aq6mZASdCAAcIeiCxoKjE28U8ajdFjjBqjB5oY4olJCYPCCe6/HuwSEzd0s0lZTQRTY+SgdhtDSozQdLctRZt2aGbuPehUCEzne6e8z7wzN9fvaNr5uOdO6dX3m2/eeSYyE0l1TQy9gKS2jFwqzsil4oxcKs7IpeKMXCrOyDuIiHdExH9ExP6I+LeIuGLonbqIiGsj4rdD77EcEbE3Ij4+9B5dRMRURPw8Ig5GxCND73Mpg0QeET+JiF9FxDeH+PiX4TPADzPzLuAYcPfA+3T1EDA59BJdRcRHgHdn5s+G3qWjzwI/zcxpYHNETA+90GJWPPKIuA8YZeYtwI6I+JuV3mG5MnNvZv7nwi+3AseH3KeLiPgYcIYL/yitehGxHngUOBoRnxh6n45OAO+NiHcC1wF/HHifRQ1xJb8N+NeFt/cDtw6ww2WJiFuAqcx8ZuhdlrLw6cS3gAeH3mUZdgOHgB8AN0fEVwfep4tfAtuBrwG/B04Ou87ihoh8E/CnhbdPAtcOsMOyRcTVwI+Azw+9SwcPAnsz8+WhF1mGDwIzmXkMeAK4feB9uvg28KXM/C5wGPjcwPssaojI/8JrnydeOdAOy7JwZdwHfD0zXxh6nw7uAL4SEQeAD0TEjwfep4s/ADsW3p4G1sKf8xTwvogYAR8CVuU3gsRKf4NKROwG3pWZD0XEd4D/ycx/XNEllikivgx8D/jdwm89nJn/MuBKnUXEgcy8beg9xomIzcBjXHhmtx74ZGb+aen/algRcTPwOBeesv8K+IfM/MuwW73ZEJFfBfwC+C/g74EPZ+bpFV1CehtZ8cjhwtcXgTuB/174HExSI4NELmnlrPoXvSS9NUYuFTdY5BGxZ6iPfbncub21ti+s/p2HvJKv6j+YS3Dn9tbavrDKd/bpulRcr6+uR8Sae6k+Ijo/NjM7P34tftViNBoNvQLz8/NMTHS/9szNzTXZY8uWLZ0fe+7cOTZu3Nj58SdOnLiclcbKzEX/cq5r8tHWkPXr1zeZOz8/32RuS5s2bWoydzn/kC7XK6+80mTuPffc02QuwOOPP95s9mJ8ui4VZ+RScUYuFWfkUnFGLhVn5FJxnSJfg6erSlowNvK1eLqqpNd0uZLfxho9XVVSt8iXPF01IvYs/ASJg30vJ+mt63Jb65Knq2bmDDADa/Pedam6Llfy3/DaU/T3A0ebbSOpd12u5P8O/CIi3sPC6aptV5LUp7FX8sx8hQsvvj0D3O7xydLa0ulbTTPzFK+9wi5pDfGON6k4I5eKM3KpOCOXinvbn/HW6lyzs2fPNpkLMDs722Tucg5QXI6XX273Y9Jb7Xz8+PEmc6HNgZlLHWjplVwqzsil4oxcKs7IpeKMXCrOyKXijFwqzsil4oxcKs7IpeKMXCrOyKXijFwqzsil4oxcKs7IpeKMXCrOyKXijFwqzsil4oxcKs7IpeJ6PZJ5YmKCycnJ8Q9cpojofeZFp06dajJ3586dTeYCnDt3rsncEydONJm7Y8eOJnOh3f+/+++/v8lcgKeeeqrZ7MV4JZeKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpuLE3w0TEO4B/BkbAGWBXZr7aejFJ/ehyJf8M8MPMvAs4BtzddiVJfRp7Jc/Mva/75VbgeLt1JPWt873rEXELMJWZz7zh9/cAexbe7nc7SW9Zp8gj4mrgR8Cb7trPzBlgBmA0GmWv20l6y8Z+Th4RVwD7gK9n5gvtV5LUpy4vvH0B+DvgGxFxICJ2Nd5JUo+6vPD2MPDwCuwiqQFvhpGKM3KpOCOXijNyqTgjl4rr9bTWVubm5prNvvHGG5vMfe6555rMbemBBx5oMvfJJ59sMhdgw4YNTeZeeeWVTeYCbNu2rfeZx44du+T7vJJLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVSckUvFGblUnJFLxRm5VJyRS8UZuVRcr0cyZyavvvpqnyMBiIjeZ1704osvNpk7Go2azIV2R1Tv27evydxNmzY1mQtw/vz5JnNb/b0AuOGGG3qfefr06Uu+zyu5VJyRS8UZuVSckUvFGblUnJFLxRm5VFynyCPi2oj4betlJPWv65X8IWCy5SKS2hgbeUR8DDgDXPqnnEtatZaMPCKuAL4FPLgy60jq27h71x8E9mbmy5e6fzwi9gB7+l5MUj/GPV2/A/hKRBwAPhARP37jAzJzJjOnM3O65TeSSLo8S17JM/OjF9+OiAOZ+cX2K0nqU+evk2fmbQ33kNSIN8NIxRm5VJyRS8UZuVSckUvFGblUXGRmb8NGo1G2OJmz1YmcAOvW9Xpg7V/deeedTeYCPPvss03mtjqhdOfOnU3mAhw9erTJ3NnZ2SZzAa6//vreZ7700kvMzs4uejeaV3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqbjeT2udnJzsbd5F8/Pzvc+86Kqrrmoy9/jx403mAkxNTTWZe8011zSZe+TIkSZzAUajUZO5t956a5O5AE8//XSTuZnpaa3S25GRS8UZuVSckUvFGblUnJFLxRm5VJyRS8V1jjwi9kbEx1suI6l/nSKPiI8A787MnzXeR1LPxkYeEeuBR4GjEfGJ9itJ6lOXK/lu4BDwA+DmiPjq698ZEXsi4mBEHOzzPnhJ/egS+QeBmcw8BjwB3P76d2bmTGZOZ+Z0xKL3x0saUJfI/wDsWHh7Gnih3TqS+rauw2N+AjwWEZ8G1gOfbLuSpD6NjTwz/wx8agV2kdSAN8NIxRm5VJyRS8UZuVSckUvFGblUXJevk3eWmU2OT56bm+t95kWtjnveuHFjk7kAre4sfP7555vMbanV340NGzY0mQuwa9eu3mfu37//ku/zSi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFWfkUnFGLhVn5FJxRi4VZ+RScUYuFReZ2duwiYmJbHHK5ZYtW3qfedHZs2ebzN22bVuTuQCHDh1qMvemm25qMvfw4cNN5gKcOXOmydyWJwRv3bq195mnTp3i/Pnzix7j65VcKs7IpeKMXCrOyKXijFwqzsil4oxcKm7JyCNiKiJ+HhEHI+KRlVpKUn/GXck/C/w0M6eBzRExvQI7SerRuMhPAO+NiHcC1wF/bL+SpD6Ni/yXwHbga8DvgZPNN5LUq3GRfxv4UmZ+FzgMfO6ND4iIPQufsx/s8z54Sf0YF/kU8L6IGAEfAt5UcWbOZOZ0Zk5HLHp/vKQBjYv8+8AMcBq4Gvin5htJ6tW6pd6Zmb8G/naFdpHUgDfDSMUZuVSckUvFGblUnJFLxRm5VJyRS8X1eiTzaDTKycnJ3uZd1PJ22YmJNv/Obd68uclcgO3btzeZe+TIkSZzT55s9y0P9957b5O59913X5O5ALt3724yNzM9kll6OzJyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4ozcqk4I5eKM3KpOCOXijNyqTgjl4rr9bTWiPhf4IWOD78G+L/ePvjKcOf21tq+sDp23p6ZWxd7R6+RL0dEHMzM6UE++GVy5/bW2r6w+nf26bpUnJFLxQ0Z+cyAH/tyuXN7a21fWOU7D/Y5uaSV4dN1qTgjl4ozcqk4I5eKM3KpuP8Hc55GZ0ESQGUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 用0填充对角线，只保留错误，重新绘制\n",
    "np.fill_diagonal(norm_conf_mx,0)\n",
    "plt.matshow(norm_conf_mx,cmap = plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   * 每行代表实际类别，每列代表预测类别\n",
    "   * 8,9列比较亮，说明许多图片被错误的分类成shuzi 8,9\n",
    "   * 8,9行也偏亮，说明8,9 经常会跟其他数字混淆\n",
    "   * 5和3也偏亮"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 结论\n",
    "   * 改进数字8,9的分类\n",
    "   * 修正数字3和5的混淆\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何优化分类器\n",
    "   * 尝试多收集这些数字的训练集\n",
    "   * 开发一些新特征来改进分类器\n",
    "   * 优化分类器算法\n",
    "   * 使用pillow或opencv对图片预处理，让显示模型更突出\n",
    "   * 分析单个错误"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAccAAAHBCAYAAAAcpXCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAS1klEQVR4nO3da4hd5dkG4Bkz5jAitRKjiAaxYpNoK6QRFTWZxBYLKpKk0IZq00JzwCpKUySJgRg1SYswaglYyY+UYATRTE8/mpZqbDA9EQlpKZlgW5pWiMZp1JJ4NtMf3/fxlcd53tlZM3vP3nuu6+/NWuvRPXtuF8zj2zk4ONgBAPy/08Z6AABoNsoRAALlCACBcgSAQDkCQKAcASDoGia35wEf1znWA1Tk+wwfN+T32ZsjAATKEQAC5QgAgXIEgEA5AkCgHAEgUI4AEChHAAiUIwAEyhEAAuUIAIFyBIBAOQJAoBwBIFCOABAoRwAIlCMABMoRAALlCACBcgSAQDkCQKAcASBQjgAQKEcACJQjAATKEQAC5QgAQddYD9CqXnjhhTTbsGFDpeuaSU9PT5qtX7++0nUArcKbIwAEyhEAAuUIAIFyBIBAOQJAoBwBIOgcHBws5cWwWZTWI0ayOlFayWBow/w8tYvOsR6goqb5cE6cOJFmmzdvTrONGzcW79vZmX80pZ/NRl83c+bMNJs7d26alUydOjXN1q5dm2bd3d2VntdGhvwQvTkCQKAcASBQjgAQKEcACJQjAATKEQCCtljlmD9/fpq1yikY7cIqR1Nrmg9n3bp1aVZa5Rju56vqasWsWbPS7OTJk2l22WWXpdnOnTvTrNGrI/fdd1+aPfjgg2k2TljlAIBaKEcACJQjAATKEQAC5QgAgXIEgKBrrAcYDa20rrF+/fpRv2dPT0+l60orMFBPkydPTrOvfvWrabZ9+/Z6jNNx8ODBNNu1a1eaHTp0KM2qrjWVrjvnnHPSbPbs2Wl29913V5plPPPmCACBcgSAQDkCQKAcASBQjgAQKEcACJQjAARtsee4e/fuNBvJLl9pf7C0r1h177Ae7r///rEeAT6mdGRVvezZsyfNli5dmmaHDx9Os9IRUqVs8eLFaTZ16tQ0W7ZsWZqV9hw5dd4cASBQjgAQKEcACJQjAATKEQAC5QgAQVuscpRWJ6oeG9NKSusq9TjOq5lWVRhfnnzyyWK+atWqNDt69GialdYu7rnnnjSbMWNGmi1fvjzNaH7eHAEgUI4AEChHAAiUIwAEyhEAAuUIAEFbrHK0i9LaxUhOF6mi6okkUE8DAwOV86onaBw8eDDN1q5dW5yH1uXNEQAC5QgAgXIEgEA5AkCgHAEgUI4AEHQOc2pF+x9pUQellYwNGzZUuq7RSusa999/f+MGaU753/03t3H9fd6zZ0+alU7z2LdvX5qVVkC2b9+eZosWLUqz7u7uNKMuhvwQvTkCQKAcASBQjgAQKEcACJQjAATKEQCCtljlKK1ADLceUVqt4NSNkxUQqxxtpnSax4oVK9Lsxz/+cZqVfrcuXLgwzR566KE0mzlzZppRmVUOAKiFcgSAQDkCQKAcASBQjgAQKEcACNpilaO0ImBVo3X09PSk2e7duxs3yPCsctDR0dHR0dfXl2YrV65Ms9LqyNSpU9Ost7c3zW677bY0o8gqBwDUQjkCQKAcASBQjgAQKEcACJQjAARdYz0A/J/SCSrz589PsyZb82AcWbRoUZrNmTMnzbZu3ZpmGzduTLPvfOc7afb3v/89zdatW5dmDM2bIwAEyhEAAuUIAIFyBIBAOQJAoBwBIGiLUzmqrgDUS+l0iXnz5qVZ6XSRRqs6y1icgjLMz3A9OJWDutmzZ0+arVq1Ks327duXZp2d+Y/s9u3b06y0qtLd3Z1mLcapHABQC+UIAIFyBIBAOQJAoBwBIFCOABC0xSrHSJRWFppptaJVlFZnSis3I2GVo2Zt/31udwMDA2n22GOPpdmmTZvSrPT9WbhwYZrt3LkzzVqMVQ4AqIVyBIBAOQJAoBwBIFCOABAoRwAIxv0qB6eu2U5BscpRM9/ncergwYNpVjp549ChQ2l2/fXXF5/Z29ubZp/73OeK1zaYVQ4AqIVyBIBAOQJAoBwBIFCOABAoRwAImmqVo7QiUMp6enoqZeRKJ5Js2LChcYP8r9LnuHv37sYN8j+sctA23n777TTbvHlzmm3cuLF432nTpqXZnXfemWbr1q0r3rcOrHIAQC2UIwAEyhEAAuUIAIFyBIBAOQJA0FSrHJ2dzfUX8uvXr0+z0qpDo1VdgRmLlYySJlvXKGmuH9TaWeVg1PT19RXzpUuXptmUKVPSbN++fWk2ffr04Qc7dVY5AKAWyhEAAuUIAIFyBIBAOQJAoBwBILDKQcMMd0JKk61rlLTqD6pVDhrmyiuvTLOXXnopzbZv355mt91224hmSljlAIBaKEcACJQjAATKEQAC5QgAgXIEgKBrrAf4b6U/9S+dLkHzaJWTTGhNhw8fLuYDAwNpVjrR4Zxzzqk8U6s7ceJEmvX396fZxo0bi/ctrWvceOONaVandY1T5s0RAALlCACBcgSAQDkCQKAcASBQjgAQKEcACJrqyKqS0p5jvXYgN2zYUJf7tgL7ikWOrBoj5557bjEv7Tnefffdadbb21t5pkbq6+tLs9I/+49+9KM0K+2OHjp0KM2G6Y6OWbNmpVnpd/bUqVOL960DR1YBQC2UIwAEyhEAAuUIAIFyBIBAOQJA0DKrHO2iHmsQpaO+ShmVWeUYI1deeWUxLx2xdPz48TTr7Mw/0tJqwaJFi9Ks9Lu19Lwnnnii0nVVn9fd3Z1mM2fOTLNly5alWUdH+d/NGKxrlFjlAIBaKEcACJQjAATKEQAC5QgAgXIEgMAqB5w6qxxNqrTKsWPHjjTbtGlTmlVdkah6XdXVkZKFCxem2fTp09NsxowZlZ7XYqxyAEAtlCMABMoRAALlCACBcgSAQDkCQGCVA06dVQ5oH1Y5AKAWyhEAAuUIAIFyBIBAOQJAoBwBIFCOABAoRwAIlCMABMoRAALlCACBcgSAQDkCQKAcASBQjgAQKEcACJQjAATKEQAC5QgAgXIEgKBrmLyzIVMAjeD7DDXy5ggAgXIEgEA5AkCgHAEgUI4AEChHAAiUIwAEyhEAAuUIAIFyBIBAOQJAoBwBIFCOABAoRwAIlCMABMoRAALlCACBcgSAQDkCQKAcASBQjgAQKEcACJQjAATKEQAC5QgAgXIEgEA5AkCgHAEgUI4AEChHAAiUIwAEyhEAAuUIAIFyBIBAOQJAoBwBIFCOABAoRwAIuobJBxsyBbSWzrEeoCLfZ/i4Ib/P3hwBIFCOABAoRwAIlCMABMoRAALlCACBcgSAQDkCQKAcASBQjgAQKEcACJQjAATKEQAC5QgAgXIEgEA5AkCgHAEgUI4AEChHAAiUIwAEyhEAAuUIAIFyBIBAOQJAoBwBIFCOABAoRwAIlCMABMoRAALlCACBcgSAQDkCQKAcASBQjgAQdI31AACML//617/S7PDhw2nW1ZVX1mc/+9kRzRR5cwSAQDkCQKAcASBQjgAQKEcACJQjAARtv8rxxhtvFPNDhw5Vuu+0adPS7OjRo5XuWbJnz540e/nllyvd86WXXkqz/fv3V7rnSHz6059Os/7+/gZOAnR0dHS8+eababZ79+7itX19fWn2wgsvpNnrr7+eZqedlr/Pvf3228V5TpU3RwAIlCMABMoRAALlCACBcgSAQDkCQNAWqxy/+93v0mz16tXFa3/9619XeuYnP/nJNBtufaQVTJ48Oc0mTJiQZp/61KfSbP78+SOaCajm97//fZr19vam2a9+9as0O3bsWOV5rr766jS74YYb0mzRokWVn3mqvDkCQKAcASBQjgAQKEcACJQjAATKEQAC5QgAQefg4GApL4aNtGXLljRbs2ZNmh0/frwe41TW09OTZp2dnZXuOXHixDRbunRppXvOmzcvzc4///xK92wj1T6osdc032dypWOiDhw4kGbf+9730uz5559Ps/feey/NzjrrrDT70pe+lGYdHR0dixcvTrMFCxakWen3WZ0M+X325ggAgXIEgEA5AkCgHAEgUI4AEChHAAiaapXjnXfeSbNp06al2YkTJ9Ls3nvvLT6zv78/zX7zm9+k2e23355mpT9hvuaaa9Ks6ioHDdeqH5RVjibxhz/8Ic2+/e1vp9nevXtHfZZHH300zW666aY0u+SSS0Z9ljFilQMAaqEcASBQjgAQKEcACJQjAATKEQCCrrEe4L+dPHkyzUqna0yaNCnN5syZU3zmd7/73eEHAxjCE088kWYrV64c9eedeeaZaVb6XXbHHXeM+iztzpsjAATKEQAC5QgAgXIEgEA5AkCgHAEgaKpTOUqz9Pb2ptmaNWvSbMKECcVnXnDBBWm2ZMmSNPva176WZhMnTkyzCy+8MM2cytEyWvWDGtencnz44YdpVjolo7Qi8ctf/jLN3nvvvTS7+OKL02zt2rVp9oUvfCHNpk+fnmYUOZUDAGqhHAEgUI4AEChHAAiUIwAEyhEAgqZa5ahqy5Ytafb0008Xr33xxRdHe5yiz3/+82l22WWXpdm1116bZpdeemmaXXHFFbUNxqmwytGC+vv702zmzJmV7jl58uQ0e/jhh9Ns+fLlaVZaBaMurHIAQC2UIwAEyhEAAuUIAIFyBIBAOQJA0BarHCUffPBBMd+1a1ea3XvvvZWeefjw4TR75513Kt2zZNKkSWlW+r//L168OM1WrFiRZqWTTMYJqxwt6Kc//Wma3XrrrZXuOWXKlDTbt29fms2aNavS86gLqxwAUAvlCACBcgSAQDkCQKAcASBQjgAQtP0qx1j44x//mGY///nP06x0asBrr71W6Z5VnX322Wm2dOnSNCutv5x33nkjmqmJWOVoQaU1qv3796fZs88+m2aPPPJImn3xi19Ms29+85tpVlqxoi6scgBALZQjAATKEQAC5QgAgXIEgEA5AkBglaNFlD6njz76KM2ee+65NNu0aVOaHThwIM3eeuutNDvttPy/t3bs2JFmHR0dHV/5yleKeROxysGwli9fnmbPPPNMmvX29qbZN77xjRHNxJCscgBALZQjAATKEQAC5QgAgXIEgEA5AkBglYMh/fWvf02zr3/962m2d+/eNJs0aVLxmT/84Q/T7Mtf/nLx2gazysGwjh8/nmY333xzmpXWqI4cOZJmkydPrm0wIqscAFAL5QgAgXIEgEA5AkCgHAEgUI4AEFjlYFStX78+zR544IHK9x3m57TRrHIwIlu2bEmzu+66K82WLFmSZk899dSIZhrHrHIAQC2UIwAEyhEAAuUIAIFyBIBAOQJAYJWDUfXvf/87zc4999zite+++26aWeUYFU31LzFTWnO48847GzhJ/fztb39Ls8svvzzNZs+enWYvvvjiiGYax6xyAEAtlCMABMoRAALlCACBcgSAQDkCQNA11gPQXg4cOJBm77//fgMnoVV9//vfT7Pbb7+9eO0nPvGJ0R6nLs4+++w0u+CCCxo4CRlvjgAQKEcACJQjAATKEQAC5QgAgXIEgMAqB6fslVdeSbM77rgjzU6ePFm873XXXVd5JtrHW2+9lWb33HNP8dpt27aN9jh1UfrnePnll9Ost7e3HuMwBG+OABAoRwAIlCMABMoRAALlCACBcgSAwCoHQ/rtb3+bZsuWLUuzP//5z2l20UUXFZ/5s5/9bNi5aH+LFy9Os61btxavveKKK9JsuDWQzMDAQJrt2LEjzX7wgx+k2ZEjR9Js06ZNaXbzzTenGaPLmyMABMoRAALlCACBcgSAQDkCQKAcASBQjgAQdA4ODpbyYkjze/XVV9PsF7/4RZotX748zd5///00O/PMM9NsuD3GefPmFfMm0jnWA1TUEt/ngwcPptlVV11VvPaMM86ofG3mJz/5SaXrrr766jQr7WtefvnllZ5HZUN+n705AkCgHAEgUI4AEChHAAiUIwAEyhEAgoavcpTWB2688cbRflzbKB0hVfpT823btqXZ0aNHK80yd+7cNHv88cfTbNasWZWe14SscoyR4Y6d6uvrS7N//vOfaXbWWWelWekYrNWrV6fZggUL0mzixIlpRsNZ5QCAWihHAAiUIwAEyhEAAuUIAIFyBICg4ascnZ35X8H/6U9/SrOx+D/Vv/nmm2n2j3/8o9I99+7dm2abN29OsyNHjqTZhx9+WGmWGTNmpNm3vvWtNFuxYkWanX766ZVmaTFWOZrUsWPH0qy/vz/Npk2blmaXXHLJiGai6VnlAIBaKEcACJQjAATKEQAC5QgAgXIEgKCpVjk+85nPpFl3d3el51144YXFfGBgIM1eeeWVNPvLX/5SaZ6qzjvvvDS75ZZb0qx0MsDChQvTbNKkSbUNNj5Z5YD2YZUDAGqhHAEgUI4AEChHAAiUIwAEyhEAgq5GP3CY1REAGHPeHAEgUI4AEChHAAiUIwAEyhEAAuUIAIFyBIBAOQJAoBwBIFCOABAoRwAIlCMABMoRAILhTuXobMgUQCP4PkONvDkCQKAcASBQjgAQKEcACJQjAATKEQCC/wDDWdnJi2faIQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_digits(instances,images_per_row=10,**options):\n",
    "    size = 28\n",
    "    images_per_row = min(len(instances),images_per_row)\n",
    "    images = [instance.reshape(size,size) for instance in instances]\n",
    "    n_rows = (len(instances) - 1) // images_per_row + 1\n",
    "    row_images = [ ]\n",
    "    n_empty = n_rows + images_per_row - len(instances)\n",
    "    images.append(np.zeros((size,size + n_empty)))\n",
    "    for row in range(n_rows):\n",
    "        rimages = images[row + images_per_row : (row +1) + images_per_row]\n",
    "        row_images.append(np.concatenate(rimages,axis=1))\n",
    "    image = np.concatenate(rimages,axis=0)\n",
    "    plt.imshow(image,cmap = matplotlib.cm.binary,**options)\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "    \n",
    "# 查看数字3和数字5 的例子\n",
    "\n",
    "cl_a,cl_b = 3,5\n",
    "X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
    "X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
    "X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
    "X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.subplot(221);\n",
    "plot_digits(X_aa[:25],images_per_row=5)\n",
    "plt.subplot(222);\n",
    "plot_digits(X_ab[:25],images_per_row=5)\n",
    "plt.subplot(223);\n",
    "plot_digits(X_ba[:25],images_per_row=5)\n",
    "plt.subplot(224);\n",
    "plot_digits(X_bb[:25],images_per_row=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多标签分类\n",
    "    1.为每个实例产生多个类别，例如照片识别多个人脸\n",
    "    2.输出多个二元标签的分类系统称为多标签分类系统"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
       "                     weights='uniform')"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# K近邻  找到离实例最近的点，把那个点的标签添在实例上\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)     # 奇数\n",
    "y_multilabel = np.c_[y_train_large,y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train,y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False,  True]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# k近邻支持多标签分类，不是所有的分类器都支持\n",
    "knn_clf.predict([some_digit])     # 5小于7,5是奇数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 评估多标签分类器方法很多，方法之一就是测量每个标签的F1分数，或者其他二元分类器指标，然后简单平均"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9767597903168503"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_knn_pred = cross_val_predict(knn_clf,X_train,y_multilabel,cv=3)\n",
    "f1_score(y_multilabel,y_train_knn_pred,average='macro')\n",
    "# 该f1分数较高，"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上面假设了所有标签都同等重要，也可以给每个标签设置一个权重，（该目标标签实例的数量），设置average='weighted'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多输出分类\n",
    "     例子：构建一个系统去除图片中的噪声，输入一张有噪声的图片，它将输入一张干净的数字图片，分类器输出是多个标签，\n",
    "     一个像素一个标签，每个标签多个值0到255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加噪声，目标讲图片还原为原始图片，，先创建训练集和测试集\n",
    "\n",
    "noise = np.random.randint(0,100,(len(X_train),784))\n",
    "X_train_mod = X_train + noise\n",
    "noise = np.random.randint(0,100,(len(X_test),784))\n",
    "X_test_mod = X_test + noise\n",
    "\n",
    "# 目标输出为原先的干净图片\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAC2CAYAAAD5uGd5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZ50lEQVR4nO3de4zU5bkH8O/jwHLZxeUqFAR0uQgoIjCgqyB7rFJrm6ZgjT3pxcSe2vQkJk38o7FrTGo9pjmJ9jS9eLqJbYiNp0qjtCdovBQFC0dxVsRLgSI3AbkjLheBZXnOHy7i7vt92dnLzO47fD9J0+XxmZnfzLz7MszzPu9r7g4REUnHBd19ASIi0j6auEVEEqOJW0QkMZq4RUQSo4lbRCQxmrhFRBKjiVtEJDG9OnpDM3sMwBQAS939wVheZWWljxgxokWsd+/e+V9gL36JmUwmiDU1NdFcFo+tX+/Tp0/euex+T58+TXNPnDgRxC64IPx7M/Z8T506FcTMjOay641dV0VFRV6PFbu22DWw59vY2Ehz+/fvH8Ri7yW7hjVr1ux392H0Bu2U77gGgKFDh/oll1zSFQ8rEti6dSv2799Pf8E6NHGb2UIAGXevNrPfm9kEd9/IckeMGIG6uroWsWHD+O8Ym8iGDBlCcwcOHBjEDh06RHOPHDkSxE6ePElz2S9ibNL76KOPghibsABg06ZNQaxv375B7KKLLqK337t3bxCL/QXIJshPPvmE5s6ZMyeI7d+/n+YOHTo0iMUm7i1btgQx9hwAYPr06UHs448/prlsPJSXl2+jye3UnnENfDpWcrlcVzy0SCCbzUb/W0e/KqkB8FTzzy8AaPHbb2Z3mVnOzHKxX0CRHqgG5xjXQMuxvW/fvmJem8hnOjpxlwPY2fzzQQDDP/8f3b3O3bPunq2srOzM9YkU0znHNdBybMf+5ShSaB39jvsIgH7NP1fgHH8B9O7dG62/4x45ciTNfffdd4NYQ0MDzZ06dWoQY19HAMDEiROD2IABA2hufX193tcwf/78IPbhhx/S3ClTpgSxrVu3BjH2fS/Av15iXwEB/KuSMWPG0Ny1a9cGsWPHjtFc9tUO+7oIAKqqqoJY7Ltz9l4cOHCA5rLXrAvlPa5FulNHB2Y9zv4zchqArV1yNSLdS+NaktDRT9xLALxqZiMBfBnANV13SSLdRuNaktChT9zu3oBPCzmvAfgXd1cFUpKncS2p6PA6bnf/CGcr8CIlQeNaUqDii4hIYjr8iTtfjY2N2L17d4vYnj17aO6ECROCGOvsA/gKlFhn3oUXXhjEWIMIwJtthg8PVoUB4CscRo0aRXOXLVsWxEaPHh3Ejh8/Tm/Pmm1iKzrmzZsXxNavX09z2QoS9j4AvJsx1oDDVviw5wsAH3zwQRCLNQzFVryInE/0iVtEJDGauEVEEqOJW0QkMZq4RUQSU/DiZHl5OWbNmtUiFtsmtby8PIgtX76c5k6bNi2IxQpl69atC2KxIiIrDsZ2EmSt+7GdBNlzY23wGzZsoLdnO+vF2tjZlrexbQbYaxYrZN54441BLHa9bI+aw4cP09yysrIgFis0x7YqEDmf6BO3iEhiNHGLiCRGE7eISGI0cYuIJEYTt4hIYgq+qgQIV5HEzkpcs2ZN3vfJWt5jrdr9+vULYtu28WMK2aqQ2GoIdnBtbFUJW9WxevXqIHbFFVfQ27OzMGOHCqxcuTKIsbMlAb7aJXY2JHPttdfSONtSIHaM3aBBg4LY+PHj875fkfONPnGLiCRGE7eISGI0cYuIJEYTt4hIYtpdnDSzXgA2N/8PAO5293di+SdPnsSOHTtaxHbt2kVzr7rqqiAW25+aFRxjxU3W1j19+vS8c7dv35537ooVK2gue27s5PajR4/S27PT48eOHUtz2X1s3ryZZAJDhgwJYrHT2FkxddKkSTSXtd1ffvnlNJftVb5//36a257CaXu1d2yLdJeOrCq5EsD/uPuPu/piRLqZxrYkoSNflVwD4KtmttrMHmv+lCJSCjS2JQkdmbjfAHCju88G0BvALV17SSLdRmNbktCRifttdz/zJXUOQND1YmZ3mVnOzHKxcxFFeqB2je19+/YV9+pEmnVk4n7czKaZWQbA1wGsbZ3g7nXunnX3LOuKE+mh2jW2WXFZpBg68h3eAwCeAGAA/uruL50rOZPJBKestz71/QzWfv3WW2/R3NmzZwcxtnID4K3WsZUTrB2fXRfA29tjv8w7d+4MYmxVyAUX8L9LL7744iAWOwl98ODBQayiooLmbty4MYjFVoqwlR6x1R9VVVVB7NVXX6W57L2MtcezQxe6ULvGtkh3affE7e7v4tPqu0hJ0diWVKgBR0QkMZq4RUQSo4lbRCQxBW8wyGQyGDhwYItYrFWbFQYvuugimvvGG28EsXnz5tHc119/PYiNGzeO5jY1NQWxWHGyPUsdWbHuzTffDGKtC7lnHDx4MIjF9jVnjxUrZLJCJNvPGwD69u0bxI4cOZJ3LiuwxnJZ0RSIn1Yv0tWefvppGl+4cGEQW7duHc2dPHlyl17TGfrELSKSGE3cIiKJ0cQtIpIYTdwiIonRxC0ikpiCrypxd5w4caJFLNYWzlrIY6s/2EoEdtgAAMyYMSOIsdUjAG/hfv/992lunz59gtiiRYtoLtuzhbXdL168OO/bL1iwgObW1tYGsXfe4ecBsFb42EoRdtr93LlzaS57L2PvD1u1E2ttjx1UIYX3i1/8gsY3bNiQ1+2feeYZGmcrm2KrMZYvX57347t7EGOHn3RFbmz1G9t+IpbLnluMPnGLiCRGE7eISGI0cYuIJEYTt4hIYgpenDx27FjQ2j1t2jSay4oBrAgJAAMGDAhisaInOxl8z549NJcVSmKn0m/bti2IPfnkkzSXFRdZrKGhgd6e7WEe27ubtcLHCi07duwIYlOmTKG57MSXXr34ELriiiuCGCuaAvy9GDp0KM1dtWoVjUvXYu3e99xzD81lY6s9xT62T/vf//53mtue+2W6Ivf6668PYo888gjNZeM4NrbbQ5+4RUQSo4lbRCQxmrhFRBKjiVtEJDGauEVEEpPXqhIzGw7gz+4+18x6A3gawGAAj7n778912759+warFGIneLPVELFc1rIeO3F87dq1QeyGG26guawNferUqTSXHeYwa9YsmpvL5YJYTU1NEJs5cya9PXsOse0Arr766iCWyWRobv/+/YPY6NGjae6oUaPyzmXt7RMnTqS5rMX+rbfeornV1dU03lGdGduljLWnsxUdMbFxnK9YyztbkcFWgrX3fmNbN/RUbX7iNrNBABYBOLOhxN0A6t39OgDfMLNwXZ5IAjS2JVX5fFXSBOB2AGcWGNcAeKr55xUAsl1/WSJFobEtSWpz4nb3Bnf//PcV5QB2Nv98EMDw1rcxs7vMLGdmuQMHDnTNlYp0sc6ObdaQJFIMHSlOHgHQr/nnCnYf7l7n7ll3zw4ZMqQz1ydSTO0a27FOXZFC60jLez2AOQD+DGAagNfOlXzq1KmgVTp2yjsroLFiIcCLlpdddhnNZW3zrLAI8L1yY63l69evD2L19fU09/777w9irPU/1m5eWVkZxGKnzL/33ntBLFZY6tevXxBje2nH4o2NjTSX/UvrpZdeorkTJkwIYmyPbgA4fvw4jXeRdo3tUhD7VwNrOY+1gN93331B7IEHHujchck5dWTiXgTgWTObC2AKgNe79pJEuo3GtiQh769K3L2m+f+3AbgJwEoAN7o7P0pGJBEa25KaDu0O6O4f4mz1XaRkaGxLCtQ5KSKSGE3cIiKJKfhBCmVlZUFbNFv1APAVCnPmzKG5rCU6ViG/9tprg1hspchf/vKXILZkyRKayw4hYG3hAF/ZwtrNY6ebb9y4MYiNHDmS5rJ4rDWdrc6JrUBh9xE7uZ2tmNm8eTPNZWLt8bFtDaRt69atC2ILFy6kueyQkNgBANddd11ejxVrN5f20yduEZHEaOIWEUmMJm4RkcRo4hYRSUxRTnlvXUicPXs2zf3HP/4RxFauXElzWWEvVlRrfco8wPeABoAnnngiiG3ZsoXmsrbsWKv2D3/4wyDG9uN++OGH6e3Zqen//Oc/ae6hQ4eCWGw/blY0ZfuiA8DOnTuDGDupPvZ4se0LWOv++++/T3PZ6yAtxYr0t9xySxBjRUiAt7fHCsPsfseMGRPEYttMdMWp5+cbfeIWEUmMJm4RkcRo4hYRSYwmbhGRxBS8OFlRURF0Lu7evZvmXnrppUEs1uG4a9euINbQ0EAyeaHl4osvprlvv/12EGOFUIAXQ2OFvQEDwuML2QHAX/rSl+jt6+rqgtgll1xCc9mBuqxgCfBOTVawBHhxiT0vANi0aVMQ+8IXvkBzWZG1qqqK5m7fvp3G5awPPviAxlkhsj0HALcnd+vWrUGM7XUPAK+88koQu/766/N+rPORPnGLiCRGE7eISGI0cYuIJEYTt4hIYjRxi4gkJq9VJWY2HMCf3X2umY3Cp4eonulJvs3deY8tgMOHD2PZsmV5XQxrF6+oqKC5vXqFlx5rY2ftt+wUa4C3dcf22J40aVIQW7BgAc1lKzVYW/eTTz5Jb19bWxvEfv3rX9NcVtH/5JNPaO7YsWODWGyPbXbK++uv8/N0r7rqqiDGTn4H+GqTPn360Fx2Kn1ndGZspyZ2Sjtz6623BrF7770379uvWLEiiP385z+nuXfccUcQe+6552gu+507H7U5cZvZIHx6+vWZWfVqAP/h7o8W8sJECk1jW1KVz1clTQBuB3BmkfQ1AP7NzN40s4cKdmUihaexLUlqc+J29wZ3//z5Vs8BqAEwC0C1mV3Z+jZmdpeZ5cwsx47GEukJOju2Y7vwiRRaR4qTq9z9sLs3AVgDYELrBHevc/esu2crKys7fZEiRdKusT1s2LDiX6EIAMu3jdXMXnH3GjN7BcC/AvgYwGoAt7r7htjtZsyY4cuXL28RY0UugO/HPXz4cJrLClqxgmOsuJiv2F7WrL091s4/derUIMb+NRJrxZ8wIZhD0NTURHOXLl0axGJt7OxgYXYwMRB/HRhWVI4VJ9l+zLF9vllxMpPJ1Lt7Nu+La6WjYzubzXoul+vow0oz1gof2/t7z549QaxU/wLNZrPI5XK0otyRvUp+CuBlACcB/Pe5BrZIYjS2JQl5T9zuXtP8/y8D0JocKRka25IaNeCIiCRGE7eISGI0cYuIJKbgBykcP348aO1ubGykuazNee/evdH7bS22+frhw4eD2IUXXkhz2UqPdevW0Vwmdgo5W4HCTjdnB0QAwD333BPEfvnLX9Lc+++/P4jF2pVZyzt7vQBg8ODBQWzcuHE0l7U8z5o1i+a+++67QSzWor9lyxYal3Sx9vavfOUrNJedKB9rjy/l0+P1iVtEJDGauEVEEqOJW0QkMZq4RUQSU/DiZFNTU1DwixXwWMt7rIg4YsSIIBYrerL7jRW/Ro8eHcROnjxJc7PZsMs61oa+atWqIMZa8dme5AAwf/78IMZORweAv/3tb0GMnaQN8EJkbC9s9vquXr2a5rLnxgrKAHDppZcGsVjbfawYKumaOXNmEHvggQdo7g9+8IMg9sc//pHm/uhHP+rchfVg+sQtIpIYTdwiIonRxC0ikhhN3CIiidHELSKSmIKvKjEzlJWVtYjV19dHc1tjq0cAYODAgUFs06ZNNPfUqVNBjJ0SD/BVHbFVMKxl/ejRozS3pqYmiLH27dgG8mwVTOzUbnbIxIYNfGvpm2++Oa/bAwA7dGPAgAE0lx2aEHtt2Eb4rBUf6PpT3qVnWr9+PY2zMR8b26VMn7hFRBKjiVtEJDGauEVEEqOJW0QkMW0WJ82sEsCfAGQAHAVwO4BHAUwBsNTdHzznA/TqhSFDhrSIscIiwAtasSLismXLgtiYMWNoLjs5PXYNrFAW2wO6qqoqiMWKKqzoyAptJ06coLdne2+/+OKLNJe1EMf2N2bt/K33Tz9j0qTwOMbY3t3sVPlDhw7RXFY8PnjwIM2NbYHQXp0d19J+sX3ta2trg9iSJUtobv/+/YMY2w6i1OXziftbAB5x9/kAdgP4JoCMu1cDqDKzCYW8QJEC0biWZLX5idvdf/u5Pw4D8G0A/9X85xcAzAHAdwQS6aE0riVleX/HbWbVAAYB2A5gZ3P4IIDhJPcuM8uZWY6tdRbpKdozrpvzPxvb+/btK9JVirSU18RtZoMB/ArAnQCOADjz5WwFuw93r3P3rLtnBw0a1FXXKtKl2juugZZjm9VDRIqhzYnbzMoALAZwr7tvA1CPT/8ZCQDTAGwt2NWJFIjGtaQsn5b37wGYAaDWzGoB/AHAd8xsJIAvA7jmnA/Qq1ewUmPbtm00t7q6Ooix08IBfmp6zNatW4PY4sWLae4dd9wRxGbMmEFzN2/eHMRiq2DYQQZLly4NYrETqy+77LIgNnv2bJr7ta99LYiNHz+e5rJVJeykewB47733glhsVQlrTWYt87HcWOt/bNVNB3RqXPdkDz7IF8Tcd999BXk8tlrkoYceCmKxlSJsK4TYdg4/+clPgtiCBQvausSSk09x8lF8ukzqM2b2VwA3AfhPd+e/5SI9mMa1pKxDm0y5+0cAnuriaxHpVhrXkgp1ToqIJEYTt4hIYgq+H3djYyN27tzZIpbJZPK+/ZVXXknjbJnhyy+/THNZazlrnQWA3/zmN0Fs6tSpNHfHjh1B7Pnnn6e5rADDrou15wP8+d50000097bbbgtiDQ0NNLf1dgRAvDWd3cesWbNo7gUXhJ8JYgVlVnCcPn06zY2148tZbFwCwK233hrEhg4dSnNZcfB3v/td3rmsEB0rOLLfxccff5zmno+FSEafuEVEEqOJW0QkMZq4RUQSo4lbRCQxmrhFRBJT8FUlp0+fDlZUxFaKsJPM2Sb7AD9IIbbJ/ujRo4NYbNXCs88+G8TYRu8AP4xh1KhRNJednM4OC4gd8PDwww8HsdgqDVbRjx1MMHHixLxuDwC7du0KYqtXr6a52Ww2iMXen2PHjgWx2OoRbezUtu9///s0zg7T2Lt3L81lK0Biq0Ji8dbYqhYA+NnPfhbE2KEdcpY+cYuIJEYTt4hIYjRxi4gkRhO3iEhiCl6c7N27d1Cwi7VUs72dx44dS3NvuOGGIJbL5WjukSNHgth3v/tdmstaam+++Waa27qVH4gXVQYPHhzEWLtx7Kg3tvf3nDlzSCawatWqIPbFL36R5r700ktBjO39DfA9sseMGUNz2b7ksbb7AQMGBDFWUAaAPn360LicNXPmTBrfvXt3EKurq+v0402ePDmIzZ07t9P3K3H6xC0ikhhN3CIiidHELSKSGE3cIiKJabM4aWaVAP4EIAPgKIDbAbwP4Ey17G53f6dgVyhSABrXkjKLtTd/lmD27wA2uvuLZvYogF0Ayt39x/k8wOTJk33RokWt75Pmsg31WRs8wFvIYyeDX3755UGMnUwN8Ar5mjVraG5VVVUQKy8vp7ns2tjKi8rKSnp71rIeW2HRt2/fIMbaygGgqakpiLHWdgCYN29e3rnsGtgp8QAwYsSIIBZ7L9lKnMmTJ9e7e9hjfw6dHdcAkM1mPbaSSaSzstkscrkcnSzb/KrE3X/r7i82/3EYgFMAvmpmq83sMTMr+JJCka6mcS0py/s7bjOrBjAIwIsAbnT32QB6A7iF5N5lZjkzy8XWbIv0BO0Z1835n43tffv2FfFKRc7Ka+I2s8EAfgXgTgBvu/uZfx/nAExone/ude6edfdsbLc7ke7W3nENtBzb2qlQukubE7eZlQFYDOBed98G4HEzm2ZmGQBfB7C2wNco0uU0riVl+XyP9z0AMwDUmlktgJcBPA7AAPzV3cOe6c/JZDKoqKhoEYt9CmfFSdYGD/B9usvKymgui8datdke16wQCgAffvhhEDt+/DjNHT9+fBBj7e2xgiMrhMb2wmYF0kwmQ3NZ4Tb2/rz55ptB7OOPP6a57AR7VowF+P7frEgMAO+802ULPTo1rkW6U5sTt7s/CuDRVuGfFuZyRIpD41pSpgYcEZHEaOIWEUmMJm4RkcRo4hYRSUzBu8NOnTqFAwcOtIixDd0BfoI3a3GO3ce4ceNo7qZNm4JY7KR5tlJj+PDhNJcdABA7NZu1dbPW/8bGRnp7dpBCdXU1zd2xY0cQi7Wms+s9ffo0zWUrcdjKGoC/F7EVN+z1fe2112guWwUjcr7RJ24RkcRo4hYRSYwmbhGRxGjiFhFJTJv7cXf6Acz2AdjW/MehAPhGy2kr1ecFpPHcxrp70Xd80thOWgrPKzquCz5xt3gws1x7N7xPQak+L6C0n1tXKtXXSc+rZ9JXJSIiidHELSKSmGJP3HVFfrxiKdXnBZT2c+tKpfo66Xn1QEX9jltERDpPX5WIiCRGE7eISGKKNnGb2WNm9n9mdl+xHrOQzGy4mb3a/HNvM/tfM1tpZnd297V1lJlVmtlzZvaCmT1jZmWl9r4VQqm9RhrbPV9RJm4zWwgg4+7VAKrMjJ6gnQozGwRgEYAzhzveDaDe3a8D8A0zC7cNTMO3ADzi7vMB7AbwTZTQ+1YIGtvJKKmxXaxP3DUAnmr++QUAc4r0uIXSBOB2AA3Nf67B2ee3AkCSC/vd/bfu/mLzH4cB+DZK630rhBqU1muksZ2AYk3c5QB2Nv98EADf4DoR7t7g7p8/3ryknp+ZVQMYBGA7Suh5FUhJvfca22ko1sR9BEC/5p8rivi4xVIyz8/MBgP4FYA7UULPq4BK/TUqmedXSmO7WBdbj7P/FJkGYGuRHrdYSuL5mVkZgMUA7nX3bSiR51Vgpf4alcTzK7WxXfCjy5otAfCqmY0E8GUA1xTpcYtlEYBnzWwugCkAXu/m6+mo7wGYAaDWzGoB/AHAd0r4fesKGttpKKmxXbTOyeZq9U0AVrg7P3QyYc0DYA6A51t9R5i0Un/fukKpv0Ya2z2PWt5FRBKT1BfyIiKiiVtEJDmauEVEEqOJW0QkMZq4RUQS8//UcKPUjzU3VQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "some_index = 5500\n",
    "plt.subplot(121);plt.imshow(X_test_mod[some_index].reshape(28,28), cmap = matplotlib.cm.binary)   # 有噪声的测试图片\n",
    "plt.subplot(122);plt.imshow(y_test_mod[some_index].reshape(28,28), cmap = matplotlib.cm.binary)    # 原来干净的图片\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD2CAYAAAD720p7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAMVUlEQVR4nO3db4hc9b3H8c/nJlmT7IqkOIRYwg1CoIQ0wbBtE5tCrkSDpQ9qbiGBtlFs2Wfio9AGRbHQPLhouVJoSlBCFNpixRQLEdOCf6LVtrPpH/VBqVyStt4KUyxJNWJt/PZBTsiymz0ze+acmck37xcsOTPfOXO+GeazvzPnnJ2fI0IAcvqPYTcAoDkEHEiMgAOJEXAgMQIOJLa46Q1ce+21sWbNmqY3A1zRpqen/xYRrdn3Nx7wNWvWqN1uN70Z4Ipm+9Sl7q+8i277Uduv2L63elsAmlQp4LZ3SloUEVskXW97bb1tAahD1RF8m6QniuVjkrbOLNqest223e50On20B6AfVQM+LumtYvkdSStnFiPiYERMRsRkqzXncz+AAaka8HclLSuWJ/p4HgANqhrMaV3cLd8o6WQt3QCoVdXTZD+RdNz2dZJulbS5vpYA1KXSCB4RZ3T+QNurkv4rIk7X2RSAelS+0CUi/q6LR9IBjCAOjgGJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJDYggNue7HtP9l+vvj5ZBONAehflemDN0j6YUR8o+5mANSryi76ZklfsP0r24/arjzHOIBmVQn4ryVtj4hPS1oi6fOzH2B7ynbbdrvT6fTbI4CKqgT89xHx12K5LWnt7AdExMGImIyIyVar1VeDAKqrEvDHbW+0vUjSFyX9ruaeANSkyufnb0n6gSRLejoifl5vSwDqsuCAR8TrOn8kHcCI40IXIDECDiRGwIHECDiQGAEHEiPgQGJcR34Feumll0rrW7duHVAnuXz44Yel9SVLlgyok4sYwYHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMc6DX4FWr15dWj969Ghp/ZFHHimtnz17dt5aRJSue+zYsdJ6k2677bbSetn/S5Juv/320vrOnTvnrV111VWl61bFCA4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiXEe/DJle9gtXHbefPPN0nq36wPGxsbqbGcgGMGBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDHOg4+ohx56qLHn3rt3b1/r79mzp/K669ev72vbWBhGcCCxngJue6Xt48XyEts/tf2y7TubbQ9AP7oG3PYKSYcljRd33SVpOiI+K+lLtq9usD8AfehlBD8naZekM8XtbZKeKJZflDQ5ewXbU7bbttudTqeOPgFU0DXgEXEmIk7PuGtc0lvF8juSVl5inYMRMRkRk61Wq55OASxYlYNs70paVixPVHwOAANQJZzTki7ML7tR0snaugFQqyrnwQ9LOmr7c5LWSfplvS1dGaanp0vr999/f2n97rvvnrf24IMPlq67eDGXP1wpeh7BI2Jb8e8pSTdLelnS9og410xrAPpV6Vd5RPy/Lh5JBzCiOEAGJEbAgcQIOJAYAQcS43xJQ1577bXS+uTknCt8F+SGG26Yt8ZpMFzACA4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiXHCtCGvvvpqo89/xx13zFvbsWNH6bpXX13+NXrj4+OldVw+GMGBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDHOgzdkampqaNt+8sknS+uHDx8urbfb7dL6Bx98UFofGxsrrWNwGMGBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDHOgzckIobdwrxuuumm0vrOnTtL67t37y6tP/bYY/PWJiYmStdFvXoawW2vtH28WP647b/Yfr74aTXbIoCquo7gtldIOizpwtd8fEbStyPiQJONAehfLyP4OUm7JJ0pbm+W9HXbJ2zvb6wzAH3rGvCIOBMRp2fc9YykbZI+JWmL7Q2z17E9Zbttu93pdGprFsDCVDmK/ouI+EdEnJP0G0lrZz8gIg5GxGRETLZafEQHhqVKwJ+1vcr2ckm3SHq95p4A1KTKabIHJD0n6Z+Svh8Rf6i3JQB16TngEbGt+Pc5SZ9oqiE0b926daX1ffv2ldbLvpNdkvbs2TNv7amnnipdF/XiSjYgMQIOJEbAgcQIOJAYAQcSI+BAYvy5KObYtWtXaf3QoUOl9SNHjsxbO3HiROm6mzZtKq1jYRjBgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxzoNjjqVLl5bWb7zxxtL6Cy+8MG9t1apVlXpCNYzgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAY58Exx0cffVRaX7ZsWeXn5jz4YDGCA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBinAfHHG+88UZpff/+/QPqBP3qOoLbvsb2M7aP2T5ie8z2o7ZfsX3vIJoEUE0vu+hflvSdiLhF0tuSdktaFBFbJF1ve22TDQKorusuekR8b8bNlqSvSPrf4vYxSVsl/bH+1gD0q+eDbLa3SFoh6c+S3irufkfSyks8dsp223a70+nU0iiAhesp4LY/Jum7ku6U9K6kC39tMHGp54iIgxExGRGTrVarrl4BLFAvB9nGJP1Y0r6IOCVpWud3yyVpo6STjXUHoC+9nCb7mqRNku6xfY+kQ5K+avs6SbdK2txgfyPrvffeK62Pj48PqJP6bdiwoa/1I6KmTtCvXg6yHZB0YOZ9tp+WdLOk/4mI0w31BqBPlS50iYi/S3qi5l4A1IxLVYHECDiQGAEHEiPgQGIEHEiMPxetyHZpff369aX1hx9+uLR+8uTJ0vrq1avnre3YsaN03W42bdpUWu/WO0YHIziQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJMZ58IqWL19eWu/21cPbt2+vs50F2bt3b2n9vvvuK61PTEzU2Q4axAgOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4lxHrwh3b4b/OzZs6X1999/v7S+dOnSeWuX83eyo16M4EBiBBxIjIADiRFwIDECDiRGwIHECDiQGOfBh6Tb35N3qwO96Bpw29dI+pGkRZLek7RL0puS/q94yF0R8VpjHQKorJdd9C9L+k5E3CLpbUnflPTDiNhW/BBuYER1DXhEfC8iflbcbEn6l6Qv2P6V7Udtz9kLsD1lu2273el0am4ZQK96Pshme4ukFZJ+Jml7RHxa0hJJn5/92Ig4GBGTETHZarVqaxbAwvR0kM32xyR9V9J/S3o7Ij4oSm1JaxvqDUCfuo7gtsck/VjSvog4Jelx2xttL5L0RUm/a7hHABX1sov+NUmbJN1j+3lJb0h6XNJvJb0SET9vrj0A/ei6ix4RByQdmHX3A820A6BOXMkGJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIzN2mue17A3ZH0qkZd10r6W+NbrQ6equG3hau7r7+MyLmfD9a4wGfs0G7HRGTA91oj+itGnpbuEH1xS46kBgBBxIbRsAPDmGbvaK3auht4QbS18A/gwMYHHbRgcQIOJDYQANezGX2iu17B7ndbmwvtv0n288XP58cdk+SZHul7ePF8hLbP7X9su07R6y3j9v+y4zXbyjzVdm+xvYzto/ZPmJ7bFTec/P01vh7bmABt71T0qKI2CLpetujNOXRBo3YjKm2V0g6LGm8uOsuSdMR8VlJX7J99Qj19hlJ357x+g1rxsnZM+Hu1ui854YyS+8gR/Btkp4olo9J2jrAbXezWV1mTB2Cczo/F/uZ4vY2XXz9XpQ0zIs3Zve2WdLXbZ+wvX9YTV1iJtyvaETec1Vm6a3DIAM+LumtYvkdSSsHuO1ufq0uM6YOWkSciYjTM+4amdfvEr09o/O/gD4laYvtDUNprDBjJtw/a0ReswsWMktvHQYZ8HclLSuWJwa87W5+HxF/LZZHdcbUUX79fhER/4iIc5J+oyG+fjNmwr1TI/aazeptIO+5Qf6Hp3VxF2mjpJMD3HY3l8OMqaP8+j1re5Xt5ZJukfT6MJq4xEy4I/OaDWuW3kF+1vyJpOO2r5N0q85/bhsV35L0A0mW9PSIzph6WNJR25+TtE7SL4fcz0wPSHpO0j8lfT8i/jCkPmbOhHuPpEOSvjoi77nZvT2n87P0NvqeG+iVbMXR15slvRgRbw9sw0kUb9Stkp6d9RkY87jS33NcqgokNkoHagDUjIADiRFwIDECDiRGwIHE/g3ATw7BzyF9WQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_mod,y_train_mod)              # 训练模型\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "plt.imshow(clean_digit.reshape(28,28), cmap = matplotlib.cm.binary)        # 绘制出预测的去噪声图片\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作业二"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30,\n",
       "                                            metric='minkowski',\n",
       "                                            metric_params=None, n_jobs=None,\n",
       "                                            n_neighbors=5, p=2,\n",
       "                                            weights='uniform'),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid=[{'n_neighbors': [2, 4, 5],\n",
       "                          'weights': ['uniform', 'distance']}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='neg_mean_squared_error', verbose=0)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# 网格搜索\n",
    "param_grid = [\n",
    "    {'n_neighbors': [2,4,5],'weights': [\"uniform\",\"distance\"]}\n",
    "]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "\n",
    "knn_search = GridSearchCV(knn_clf,param_grid,cv=5,scoring='neg_mean_squared_error',return_train_score=True)  \n",
    "\n",
    "knn_search.fit(y_train_knn_pred,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=4, p=2,\n",
       "                     weights='uniform')"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 取得最优参数集合的模型\n",
    "knn_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model = knn_search.best_estimator_  \n",
    "# 用最优参数模型进行预测\n",
    "final_predictions = cross_val_predict(final_model,X_train,y_multilabel,cv=3)\n",
    "# 获得精度值\n",
    "precision_score(y_test, final_predictions) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 作业三 作业四未处理"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
